{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "np.random.seed(123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "credit_card_default = pd.read_csv('~/开发/AI相关/特征工程/Feature-Engineering-Made-Easy-master/data/credit_card_default.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>LIMIT_BAL</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>167484.322667</td>\n",
       "      <td>129747.661567</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>50000.00</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>240000.00</td>\n",
       "      <td>1000000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>SEX</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>1.603733</td>\n",
       "      <td>0.489129</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.00</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>EDUCATION</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>1.853133</td>\n",
       "      <td>0.790349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.00</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>MARRIAGE</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>1.551867</td>\n",
       "      <td>0.521970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.00</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>AGE</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>35.485500</td>\n",
       "      <td>9.217904</td>\n",
       "      <td>21.0</td>\n",
       "      <td>28.00</td>\n",
       "      <td>34.0</td>\n",
       "      <td>41.00</td>\n",
       "      <td>79.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.016700</td>\n",
       "      <td>1.123802</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_2</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.133767</td>\n",
       "      <td>1.197186</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_3</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.166200</td>\n",
       "      <td>1.196868</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_4</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.220667</td>\n",
       "      <td>1.169139</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_5</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.266200</td>\n",
       "      <td>1.133187</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_6</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>-0.291100</td>\n",
       "      <td>1.149988</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT1</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>51223.330900</td>\n",
       "      <td>73635.860576</td>\n",
       "      <td>-165580.0</td>\n",
       "      <td>3558.75</td>\n",
       "      <td>22381.5</td>\n",
       "      <td>67091.00</td>\n",
       "      <td>964511.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT2</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>49179.075167</td>\n",
       "      <td>71173.768783</td>\n",
       "      <td>-69777.0</td>\n",
       "      <td>2984.75</td>\n",
       "      <td>21200.0</td>\n",
       "      <td>64006.25</td>\n",
       "      <td>983931.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT3</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>47013.154800</td>\n",
       "      <td>69349.387427</td>\n",
       "      <td>-157264.0</td>\n",
       "      <td>2666.25</td>\n",
       "      <td>20088.5</td>\n",
       "      <td>60164.75</td>\n",
       "      <td>1664089.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT4</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>43262.948967</td>\n",
       "      <td>64332.856134</td>\n",
       "      <td>-170000.0</td>\n",
       "      <td>2326.75</td>\n",
       "      <td>19052.0</td>\n",
       "      <td>54506.00</td>\n",
       "      <td>891586.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT5</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>40311.400967</td>\n",
       "      <td>60797.155770</td>\n",
       "      <td>-81334.0</td>\n",
       "      <td>1763.00</td>\n",
       "      <td>18104.5</td>\n",
       "      <td>50190.50</td>\n",
       "      <td>927171.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>BILL_AMT6</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>38871.760400</td>\n",
       "      <td>59554.107537</td>\n",
       "      <td>-339603.0</td>\n",
       "      <td>1256.00</td>\n",
       "      <td>17071.0</td>\n",
       "      <td>49198.25</td>\n",
       "      <td>961664.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT1</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5663.580500</td>\n",
       "      <td>16563.280354</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.00</td>\n",
       "      <td>2100.0</td>\n",
       "      <td>5006.00</td>\n",
       "      <td>873552.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT2</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5921.163500</td>\n",
       "      <td>23040.870402</td>\n",
       "      <td>0.0</td>\n",
       "      <td>833.00</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>5000.00</td>\n",
       "      <td>1684259.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT3</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5225.681500</td>\n",
       "      <td>17606.961470</td>\n",
       "      <td>0.0</td>\n",
       "      <td>390.00</td>\n",
       "      <td>1800.0</td>\n",
       "      <td>4505.00</td>\n",
       "      <td>896040.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT4</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>4826.076867</td>\n",
       "      <td>15666.159744</td>\n",
       "      <td>0.0</td>\n",
       "      <td>296.00</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>4013.25</td>\n",
       "      <td>621000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT5</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>4799.387633</td>\n",
       "      <td>15278.305679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>252.50</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>4031.50</td>\n",
       "      <td>426529.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PAY_AMT6</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5215.502567</td>\n",
       "      <td>17777.465775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>117.75</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>4000.00</td>\n",
       "      <td>528666.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>default payment next month</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>0.221200</td>\n",
       "      <td>0.415062</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              count           mean            std       min  \\\n",
       "LIMIT_BAL                   30000.0  167484.322667  129747.661567   10000.0   \n",
       "SEX                         30000.0       1.603733       0.489129       1.0   \n",
       "EDUCATION                   30000.0       1.853133       0.790349       0.0   \n",
       "MARRIAGE                    30000.0       1.551867       0.521970       0.0   \n",
       "AGE                         30000.0      35.485500       9.217904      21.0   \n",
       "PAY_0                       30000.0      -0.016700       1.123802      -2.0   \n",
       "PAY_2                       30000.0      -0.133767       1.197186      -2.0   \n",
       "PAY_3                       30000.0      -0.166200       1.196868      -2.0   \n",
       "PAY_4                       30000.0      -0.220667       1.169139      -2.0   \n",
       "PAY_5                       30000.0      -0.266200       1.133187      -2.0   \n",
       "PAY_6                       30000.0      -0.291100       1.149988      -2.0   \n",
       "BILL_AMT1                   30000.0   51223.330900   73635.860576 -165580.0   \n",
       "BILL_AMT2                   30000.0   49179.075167   71173.768783  -69777.0   \n",
       "BILL_AMT3                   30000.0   47013.154800   69349.387427 -157264.0   \n",
       "BILL_AMT4                   30000.0   43262.948967   64332.856134 -170000.0   \n",
       "BILL_AMT5                   30000.0   40311.400967   60797.155770  -81334.0   \n",
       "BILL_AMT6                   30000.0   38871.760400   59554.107537 -339603.0   \n",
       "PAY_AMT1                    30000.0    5663.580500   16563.280354       0.0   \n",
       "PAY_AMT2                    30000.0    5921.163500   23040.870402       0.0   \n",
       "PAY_AMT3                    30000.0    5225.681500   17606.961470       0.0   \n",
       "PAY_AMT4                    30000.0    4826.076867   15666.159744       0.0   \n",
       "PAY_AMT5                    30000.0    4799.387633   15278.305679       0.0   \n",
       "PAY_AMT6                    30000.0    5215.502567   17777.465775       0.0   \n",
       "default payment next month  30000.0       0.221200       0.415062       0.0   \n",
       "\n",
       "                                 25%       50%        75%        max  \n",
       "LIMIT_BAL                   50000.00  140000.0  240000.00  1000000.0  \n",
       "SEX                             1.00       2.0       2.00        2.0  \n",
       "EDUCATION                       1.00       2.0       2.00        6.0  \n",
       "MARRIAGE                        1.00       2.0       2.00        3.0  \n",
       "AGE                            28.00      34.0      41.00       79.0  \n",
       "PAY_0                          -1.00       0.0       0.00        8.0  \n",
       "PAY_2                          -1.00       0.0       0.00        8.0  \n",
       "PAY_3                          -1.00       0.0       0.00        8.0  \n",
       "PAY_4                          -1.00       0.0       0.00        8.0  \n",
       "PAY_5                          -1.00       0.0       0.00        8.0  \n",
       "PAY_6                          -1.00       0.0       0.00        8.0  \n",
       "BILL_AMT1                    3558.75   22381.5   67091.00   964511.0  \n",
       "BILL_AMT2                    2984.75   21200.0   64006.25   983931.0  \n",
       "BILL_AMT3                    2666.25   20088.5   60164.75  1664089.0  \n",
       "BILL_AMT4                    2326.75   19052.0   54506.00   891586.0  \n",
       "BILL_AMT5                    1763.00   18104.5   50190.50   927171.0  \n",
       "BILL_AMT6                    1256.00   17071.0   49198.25   961664.0  \n",
       "PAY_AMT1                     1000.00    2100.0    5006.00   873552.0  \n",
       "PAY_AMT2                      833.00    2009.0    5000.00  1684259.0  \n",
       "PAY_AMT3                      390.00    1800.0    4505.00   896040.0  \n",
       "PAY_AMT4                      296.00    1500.0    4013.25   621000.0  \n",
       "PAY_AMT5                      252.50    1500.0    4031.50   426529.0  \n",
       "PAY_AMT6                      117.75    1500.0    4000.00   528666.0  \n",
       "default payment next month      0.00       0.0       0.00        1.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_card_default.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LIMIT_BAL</th>\n",
       "      <th>SEX</th>\n",
       "      <th>EDUCATION</th>\n",
       "      <th>MARRIAGE</th>\n",
       "      <th>AGE</th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>PAY_5</th>\n",
       "      <th>...</th>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <th>PAY_AMT6</th>\n",
       "      <th>default payment next month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>20000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>689</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>120000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3272</td>\n",
       "      <td>3455</td>\n",
       "      <td>3261</td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>90000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>14331</td>\n",
       "      <td>14948</td>\n",
       "      <td>15549</td>\n",
       "      <td>1518</td>\n",
       "      <td>1500</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>50000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>28314</td>\n",
       "      <td>28959</td>\n",
       "      <td>29547</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019</td>\n",
       "      <td>1200</td>\n",
       "      <td>1100</td>\n",
       "      <td>1069</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>50000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>20940</td>\n",
       "      <td>19146</td>\n",
       "      <td>19131</td>\n",
       "      <td>2000</td>\n",
       "      <td>36681</td>\n",
       "      <td>10000</td>\n",
       "      <td>9000</td>\n",
       "      <td>689</td>\n",
       "      <td>679</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LIMIT_BAL  SEX  EDUCATION  MARRIAGE  AGE  PAY_0  PAY_2  PAY_3  PAY_4  \\\n",
       "0      20000    2          2         1   24      2      2     -1     -1   \n",
       "1     120000    2          2         2   26     -1      2      0      0   \n",
       "2      90000    2          2         2   34      0      0      0      0   \n",
       "3      50000    2          2         1   37      0      0      0      0   \n",
       "4      50000    1          2         1   57     -1      0     -1      0   \n",
       "\n",
       "   PAY_5  ...  BILL_AMT4  BILL_AMT5  BILL_AMT6  PAY_AMT1  PAY_AMT2  PAY_AMT3  \\\n",
       "0     -2  ...          0          0          0         0       689         0   \n",
       "1      0  ...       3272       3455       3261         0      1000      1000   \n",
       "2      0  ...      14331      14948      15549      1518      1500      1000   \n",
       "3      0  ...      28314      28959      29547      2000      2019      1200   \n",
       "4      0  ...      20940      19146      19131      2000     36681     10000   \n",
       "\n",
       "   PAY_AMT4  PAY_AMT5  PAY_AMT6  default payment next month  \n",
       "0         0         0         0                           1  \n",
       "1      1000         0      2000                           1  \n",
       "2      1000      1000      5000                           0  \n",
       "3      1100      1069      1000                           0  \n",
       "4      9000       689       679                           0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_card_default.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LIMIT_BAL                     0\n",
       "SEX                           0\n",
       "EDUCATION                     0\n",
       "MARRIAGE                      0\n",
       "AGE                           0\n",
       "PAY_0                         0\n",
       "PAY_2                         0\n",
       "PAY_3                         0\n",
       "PAY_4                         0\n",
       "PAY_5                         0\n",
       "PAY_6                         0\n",
       "BILL_AMT1                     0\n",
       "BILL_AMT2                     0\n",
       "BILL_AMT3                     0\n",
       "BILL_AMT4                     0\n",
       "BILL_AMT5                     0\n",
       "BILL_AMT6                     0\n",
       "PAY_AMT1                      0\n",
       "PAY_AMT2                      0\n",
       "PAY_AMT3                      0\n",
       "PAY_AMT4                      0\n",
       "PAY_AMT5                      0\n",
       "PAY_AMT6                      0\n",
       "default payment next month    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credit_card_default.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.7788\n",
       "1    0.2212\n",
       "Name: default payment next month, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = credit_card_default.drop('default payment next month', axis=1)\n",
    "y = credit_card_default['default payment next month']\n",
    "y.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "def get_best_model_and_accuracy(model, params, X, y):\n",
    "    grid = GridSearchCV(model, #要搜索的模型\n",
    "                       params, #要尝试的参数\n",
    "                       error_score = 0.)\n",
    "    grid.fit(X, y)\n",
    "    \n",
    "    # our classical metric for performance\n",
    "    print(\"Best Accuracy: {}\".format(grid.best_score_))\n",
    "    # the best parameters that caused the best accuracy\n",
    "    print(\"Best Parameters: {}\".format(grid.best_params_))\n",
    "    # the average time it took a model to fit to the data (in seconds)\n",
    "    print(\"Average Time to Fit (s): {}\".format(round(grid.cv_results_['mean_fit_time'].mean(), 3)))\n",
    "    # the average time it took a model to predict out of sample data (in seconds)\n",
    "    # this metric gives us insight into how this model will perform in real-time analysis\n",
    "    print(\"Average Time to Score (s): {}\".format(round(grid.cv_results_['mean_score_time'].mean(), 3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 选择合适模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "\n",
    "# Set up some parameters for our grid search\n",
    "# We will start with four different machine learning models\n",
    "# logistic regression, KNN, Decision Tree, and Random Forest\n",
    "lr_params = {'C':[1e-1, 1e0, 1e1, 1e2], 'penalty':['l1', 'l2']}\n",
    "knn_params = {'n_neighbors': [1, 3, 5, 7]}\n",
    "tree_params = {'max_depth': [None, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21]}\n",
    "forest_params = {'n_estimators': [10, 50, 100], 'max_depth': [None, 1, 3, 5, 7]}\n",
    "\n",
    "\n",
    "# instantiate the four machine learning models\n",
    "lr = LogisticRegression()\n",
    "knn = KNeighborsClassifier()\n",
    "d_tree = DecisionTreeClassifier()\n",
    "forest = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8095666666666667\n",
      "Best Parameters: {'C': 0.1, 'penalty': 'l1'}\n",
      "Average Time to Fit (s): 0.474\n",
      "Average Time to Score (s): 0.071\n"
     ]
    }
   ],
   "source": [
    "get_best_model_and_accuracy(lr, lr_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.7602333333333333\n",
      "Best Parameters: {'n_neighbors': 7}\n",
      "Average Time to Fit (s): 0.02\n",
      "Average Time to Score (s): 0.796\n"
     ]
    }
   ],
   "source": [
    "get_best_model_and_accuracy(knn, knn_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classifier__n_neighbors': [1, 3, 5, 7]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8008\n",
      "Best Parameters: {'classifier__n_neighbors': 7}\n",
      "Average Time to Fit (s): 0.041\n",
      "Average Time to Score (s): 6.52\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# construct pipeline parameters based on the parameters\n",
    "# for KNN on its own\n",
    "knn_pipe_params = {'classifier__{}'.format(k): v for k, v in knn_params.items()}\n",
    "print(knn_pipe_params)\n",
    "\n",
    "# KNN requires a standard scalar due to using Euclidean distance as\n",
    "# the main equation for predicting observations\n",
    "knn_pipe = Pipeline([('scale', StandardScaler()), ('classifier', knn)])\n",
    "\n",
    "# quick to fit, very slow to predict\n",
    "get_best_model_and_accuracy(knn_pipe, knn_pipe_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8202666666666667\n",
      "Best Parameters: {'max_depth': 3}\n",
      "Average Time to Fit (s): 0.23\n",
      "Average Time to Score (s): 0.003\n"
     ]
    }
   ],
   "source": [
    "get_best_model_and_accuracy(d_tree, tree_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8189\n",
      "Best Parameters: {'max_depth': 7, 'n_estimators': 100}\n",
      "Average Time to Fit (s): 1.01\n",
      "Average Time to Score (s): 0.043\n"
     ]
    }
   ],
   "source": [
    "get_best_model_and_accuracy(forest, forest_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a25935450>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd4AAAFUCAYAAACKtV19AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde1hTR/oH8G+4BIQi4RosFxVUKqjQBcVFLQheq0BBwdTCKq0U0XZb67UK1q62KOx6qVVrRaokoqLSgre6xWLBtXj7iVrd4qWiYgVWELxhgSS/P1jOGhMgE2KU8H76nOdp5sycOYfEvJk5c2Z4NTU1chBCCCFEJwye9wkQQgghnQkFXkIIIUSHKPASQgghOkSBlxBCCNEhCryEEEKIDlHgJYQQQnSIAi8hhBCiQ88k8CYkJGDSpEkq9/Xv3x9r167lXo8bNw4CgQCpqalKeadOnQqBQIC5c+eqPHb//v0hEAha3MaNG9fmuT55DGtra/Tt2xezZs3C/fv3Veb39/eHjY0Nrly5wnTdhBBCtONf//oXRCIR+vbtC4FAgG3btrVZ5sKFC3j99dfh4OCAvn37YsWKFZDLFaexyMnJgZ+fH+zt7eHn54e9e/c+k/N/IVq8Tk5O2LZtm8Ifobq6GgcPHoSTk1OL5fLz81FSUoKSkhLs2bMHAPDjjz9yaRKJRK36582bh5KSEvzyyy/YsGEDfvjhB3zyySdK+U6fPo2qqiqIRCKIxWLGqySEEKINDx8+hIeHB5YvX44uXbq0mf/evXsIDw+Hvb09fvzxRyxfvhxr167Fl19+yeU5ceIE3n77bURGRqKwsBCRkZGYOnUqTp06pfXzN9L6ETUwYsQI7N+/H4WFhXjttdcAADt37oSPjw94PF6L5Wxtbbn///333wEANjY2EAqFTPVbWFhwZV5++WW88cYb+Pnnn5XyicViTJw4EePGjcPUqVORlJQEI6MX4k9ICCGdxqhRozBq1CgAwIwZM9rMv2vXLtTV1WHDhg3o0qULPDw8cOnSJaxfvx7vvfceeDweNmzYgGHDhmHOnDkAAHd3dxQWFmLDhg3YvHmzVs//hWjxGhsbQyQSKbRQJRIJYmJidH4uN2/exI8//ggfHx+F9EePHiE7OxtRUVH485//jC5duuD777/X+fkRQghhc+LECe57u1lwcDBu376N69evAwBOnjyJoKAghXLBwcE4fvy41s/nhQi8ABAdHY19+/bh3r17OHPmDG7evImwsDCd1L106VI4OjrCwcEB/fv3h6WlJRYvXqyQJzs7G46OjvDy8gKPx0NUVBR1NxNCSAdQWVkJOzs7hbTm15WVlQCAiooKlXma92vTC9NP6u7ujn79+mHPnj04f/48IiIiYGZmppO6Z86ciZiYGMjlcpSVlWHp0qWIiorCvn37YGDQ9NtEIpEoDJwSiURYtWoVbt++jW7dujHXadZQxZR/iW8icx3/rC9jyt/dWMBcx/jGrkz5P6//lbmOBlkDU34DniFzHcYGbGWMNKmDscxXBmy3TACAdcWTlm/ktMzQQKZBKTZyOfuZ3ZK2fa/vSQ2t3MZqiZmM7dobNajDWP7s/75r+LXMZbJ+2aG1+lm+/x4Z22ilzqdvWzaPKXoyXVWe1m53auqFCbxAU6s3LS0NpaWlyM7O1lm91tbWcHV1BQC4ubnBzMwMI0eORGFhIQICAnDp0iUUFRXhxIkTWLp0KVdOKpVi27Zt3D0BQgghapBJdVqdvb29Usv1zp07AP7X8hUKhSrzPN0K1oYXpqsZAMLDw3H16lU4OjrC19f3uZ2HoWFTy+TRo0cAmgZV+fr64ujRoygsLOS2BQsWQCKRKA1JJ4QQ0gq5TP1NCwYNGoSff/4Zjx8/5tLy8/PRrVs3dO/eHQAwcOBA5OfnK5TLz8+Hn5+fVs7hSc+sxXvv3j2cO3dOIc3S0rLVMhYWFrh48SIX+HTl/v37qKiogFwux61bt7B48WLY2NjAz88PDQ0N2LFjB2bPng0PDw+FctbW1khJSUFBQQECAgIAtHzdzW8uIYR0eoxd9k978OABfvvtt/8eSoaysjKcO3cOVlZWcHZ2xqefforTp08jNzcXADBx4kSsWLECM2bMwJw5c3DlyhWsXr0a8+bN47qSp0+fjtdffx0rV67E+PHjsW/fPhQWFj6TQbTPLPD+/PPP3KNBzUJDQ9ss11ZwfhZSUlKQkpICoOkRpT/96U/49ttvYW1tjdzcXNy5c0fluTs4OMDPzw9isZgLvC1dd0ZGhkIa6z3bJaeWMeUHgEbfRUz574O9++eQ0QOm/EONezDXIWW8c6lJNw6P8W6nJnUYMtaxRYM7sJrcjTLQwbXrgpGxDu49M+bnMZfQjR546bnWL29nS/bMmTMICQnhXicnJyM5ORlvvvkmNmzYgPLycly7do3bb2lpiW+//RZz5szB8OHDIRAIMHPmTLz33ntcHj8/P6Snp2PZsmVITk5Gz549kZ6e/kx6X3k1NTUv5idDz6V4JTDl1yTwJuog8N6V1zPl78Jj/63XWQMva0AE2AOvJnW8sIFXo58dbHQxeE0XHmrwbz3lbJrW6u9y/4baeessXLRW74vihRpcRQghpBPQwcjtF5leB96srCzMmjVL5T5nZ2cUFRXp+IwIIYRAyvaIoL7R68A7duzYFvvnaapHQgh5Tto5uKqj0+voY2FhAQsLi+d9GoQQQp7Q3sFVHZ1eB15CCCEvIGrxEkIIITpELV5CCCFEh3Q8ZeSLhgIvIYQQ3aIWLyGEEKJDdI+XPA+sS/axTv8IAMtOfcaUP+xP77Wd6SlTGqyY8v/HiH0uH9YPqSb/pFnLSDWYkoj1ycW7Giy/x1pCpsGUhg2MZXQ1NV4d49VrMqsU+yxq7LWwviea1GEHY+YyWtXJW7xqzf6WkJAAgUCgtI0YMQIA0L9/fy5NKBTC09MTb731Fg4ePKhwnOvXr0MgEODMmTNKdYwbNw5z585VSDt37hxiY2PRp08fCIVCvPrqq0hISMCFCxeUyotEIlhbWyutLuHh4aHy3Ju3sLAwLt/69esVyv766694++230bt3b9jb28PLywuJiYmoqalRyDdmzBgIBALs2bNHIT0jIwMuLvo33RkhhLSLTKb+pofUnnY1MDAQJSUlCtuuXbu4/fPmzUNJSQlOnTqFzZs3w8XFBdHR0UrBVF3ff/89RowYgYcPH2Ljxo04fvw40tPT4eDggE8//VQhb3l5OQoKCjBjxgylxQgKCgq4883KygIA/PTTT1za1q1bVdZ/8uRJBAcHo66uDpmZmTh9+jSWL1+OQ4cOYcyYMaitVVxI2tTUFEuXLkV9PdvcxYQQ0tnIZQ1qb/pI7V48ExMTCIXCFvdbWFhw+52dnTF48GC4u7vjww8/REhIiNKKPa159OgRZs6ciaCgIOzYsYNL79GjB1599VWlFmdmZiaCg4MRHx+PgQMHorq6GtbW1gCaVhtqZmVlxaW1di0ymQwzZ86Eu7s7tm3bBgMDA+66+vfvD19fX3z++edYsWIFV2bChAn4/vvvkZ6ejunTp6t9rYQQ0unoaUtWXc90oZGYmBgIBALs3buXqdzhw4dRVVXV4jzLAoGA+3+5XA6JRIKoqCg4OzvDx8dHIVhrori4GJcuXcL777/PBd1mTk5OiIiIUOpWtrCwwNy5c5Gamop79+61q35CCNFrrS18//Smh9QOvHl5eXB0dFTYPvnkk1bLGBoaolevXigtLWU6qeYFjt3d3dvMW1hYiLt372L06NEAmu71SiQSpvqeduXKFQBAnz59VO5/5ZVXcOfOHdy9e1ch/e2334aFhQW++OKLdtVPCCF6TSZVf9NDanc1+/v7Y82aNQpp6ixaL5fLweOxjbqTy9Uf1SeRSBAeHg4+nw8ACAsLw7x583Dq1Kl2L2Dc0nk3n9/T+/l8PhITE/HXv/4V06ZNa1fdhBCit/S0JasutVu8ZmZmcHV1VdhsbGxaLSOVSnHlyhV0794dwP8Ctaqu2NraWnTt2hUA4ObmBgAoKSlp9fg1NTXIzc3Fli1bYGNjAxsbG/Ts2RN1dXVKg6xY9OrVC0DTqGZVSkpKYGNjo/KHx4QJE9CnTx8sX75c4/oJIUSv0ajmZycjIwO1tbXcIzsCgQA2NjYoLi5WyHfv3j1cu3aNC3hBQUGwsbHBqlWrVB63eXDVrl27YGtri6NHj6KwsJDb1qxZg2+//RYPHz7U6Ly9vb3Rq1cvfPnll5A99caXlZUhOzsbEydOVNki5vF4+PTTTyEWi3Hp0iWN6ieEEL3Wye/xqt3V/Mcff6CiokIhzdDQkBs1fP/+fVRUVKChoQG3bt3Cd999h6+//hpxcXEYOnQoV2bmzJlYtWoVHBwcMGjQIFRXVyMlJQXW1tZ44403AADm5ub44osvMHXqVERFRSEhIQGurq64e/cu9u3bh3PnziErKwtisRihoaHw8PBQOK9evXohKSkJ2dnZiImJYf6jGBgY4Msvv0RERATeeustfPTRR+jWrRvOnz+PxYsXo0ePHli4cGGL5QMCAhAYGIi0tDSuC5wQQsh/6WlLVl1qB94jR44oDXZ6+eWXcfHiRQBASkoKUlJSwOfzYWtrC29vb4jFYrz++usKZT744AOYm5tjzZo1uH79OiwtLTF48GDs27cPXbp04fKNGzcOP/zwA1atWoV3330XtbW1ePnll/HnP/8Zn376KYqLi3Hu3DmkpqYqnSufz8fYsWMhFos1CrwAMHjwYOTl5SElJQUikQj37t1Dt27dEBoaijlz5rR5f3vJkiUICAigwEsIIU/r5IGXV1NTo6sZ3cgT3hnA9oPAnmfKXEep7AFT/pz/+5K5ji4vD2PKP/flAOY6WO+HaDIVICtNpuljVY1G5jKsf6sX9etPk3tghjp4T1hr0GRMriFjfk2+wFmn/QSAz85+rUFNqvHPfqt23nqvcK3V+6KguZoJIYTolpT9R6U+ocBLCCFEtzp5VzMFXkIIIbqlp6OV1UWBlxBCiG5Ri5cQQgjRIWrxEkIIITpELV5CCCFEhyjwEkIIITpEXc2EEEKIDlGLlzwP4xu7MuU/ZMQ2CxUATGmwYsrPOgsVANT9XsiU/z3f+cx1SDWam+fFw2Oc96iHnH260Rfx60xX794fPLaaNPlbsc4qpcnMVbqYfYz/vP9NaanFm5aWhi+++AIVFRV45ZVXkJycDH9/f5V5ExISsH37dqV0MzMz/P777wCa1ncPCQlRynPixIkW12fXBAVeQgghuqWFFm92djYWLFiAf/zjHxg8eDDS0tIQGRmJoqIiODs7K+Vfvnw5lixZopA2evRolYG6qKgIVlb/a7g0LwakLc90WcCO4uzZs7C2tsbo0aNV7j969ChEIhHc3Nxgb28PLy8v/OUvf8FPP/3E5SksLIRAIFC50fKAhBDyBGmj+lsL1q1bh8mTJ2PKlClwd3dHamoqhEIh0tPTVea3tLSEUCjktmvXrqG0tBRTpkxRymtnZ6eQ19CQta+jdRR40bRu8DvvvIN///vfKCkpUdj3zTffIDQ0FJaWlkhPT8eJEyewadMmDB06FPPnK3ebFhUVoaSkRGFzc3PT1aUQQsiLr7WF75/eVKivr0dxcTGCgoIU0oOCgnD8+HG1TmHr1q3o27cv/Pz8lPYFBgbC3d0doaGhKCgoYL++NnT6rua6ujrs2rULBw4cQF1dHcRiMZYtWwagadH7+fPnY/r06fj888+5Mj169MCgQYMQFxendDw7OzvY2Njo7PwJIaTDkbfvHnNVVRWkUins7OwU0u3s7FBZWdlm+draWuTk5CApKUkh3cHBAStXrsSf/vQn1NfXY+fOnQgLC8O+ffswZMiQdp3zkzp94M3JyYGzszP69euHSZMmITY2Fp988gmMjY2Rk5OD+vp6fPDBByrL8ni6WICOEEL0jJZGNT/9HSyXy9X6Xs7KyoJUKoVIJFJI7927N3r37s29HjRoEG7cuIG1a9dqNfB2+q7mjIwM7o8/dOhQdOnSBQcOHAAAXL16FV27doVQKOTyf//993B0dOS2Y8eOKRxvwIABCvs9PDx0dzGEENIRtLOr2cbGBoaGhkqt2zt37ii1glXZunUrQkNDFQZQtcTHxwe//fabetelpk7d4v3tt99w/PhxbN68GUDTr6eoqCiIxWKEhYWpLDNs2DAUFhaipqYGQUFBkEoVHxjYu3cvBAIB99rAoNP/tiGEEEXtfJyIz+fD29sb+fn5eOONN7j0/Px8hIaGtlr29OnT+OWXX5CcnKxWXefPn1dofGlDpw68GRkZkEql6NevH5cm/++9h7KyMri5ueHevXsoLy+Hg4MDAMDc3Byurq6oqqpSeczu3bvTPV5CCGmNFrqaZ86cifj4ePj4+MDPzw/p6ekoLy9HbGwsACA+Ph4AsHHjRoVyW7ZsgZubG4YOHap0zPXr18PFxQV9+/ZFfX09srKysH//fmRkZLT7fJ/UaQNvY2Mjtm/fjk8++UTpMaL4+Hhs27YNb731FpYsWYKVK1ciJSXlOZ0pIYTomXYOrgKAiIgIVFdXIzU1FRUVFejbty+ysrLg4uICoKnx9LT79+8jOzsb8+bNU3kvuKGhAUlJSbh9+zZMTU25Y44aNard5/ukTht4Dx06hKqqKkyZMgXW1tYK+yZMmIDNmzdj7ty5SE5Oxty5c1FTU4O33noLPXr0QE1NDXJycgBA6fmu//znP2hsVHz2zMrKCnw++yxEhBCil7Q0uGratGmYNm2ayn379+9XSrOwsMCtW7daPN4HH3zQ4mBabeq0gVcsFmPYsGFKQRcA3njjDSxZsgRHjhzBtGnT0KdPH6xbtw6xsbG4d+8erKys4Ovri+3btyvNejJ48GCl43333XcIDAxUSPu8/lem8x1q3IMpPwD8x4ht1PXclwOY62CdAvLLUyuY62iQsJWRlv7OXIf84R9M+WWPWn6wvyXSe2wTCOafcmKuw4CxJSHVo5H5OfzHTPkNdXDtmozwYJ1aVBOzeGyfd62juZo7px07drS4r7lV2+y1117Da6+91urxhg0bplCGEEKIanKpJrNY649OG3gJIYQ8J9TiJYQQQnSI1uMlhBBCdEimH0t9aooCLyGEEN2irmZCCCFEhyjwEkIIITqkhQk0OjIKvIQQQnSLWryEEEKIDtHgKvI8NMgamPJLwf5BZX1zNZllh/W8WGehAgDjaLbZsXhHMpnrkJeWspd58JApv9E9tvxjet/Hz9u6MJVhJgekOpgpiZUu2kOGOrhuTeow0sF51dSZMJdx0eYJ0ONEhBBWrEFXE8886OLFDLpE/8kbO/fMVXqxWGxCQgIEAgEEAgFsbW3h5eWFxMREPHyo+OUoEolgbW2N/Px8hfQlS5bA09MTtbW1CukXLlyAvb09tyBCS0pLSzFz5kwMGDAADg4O8PLywtKlS/H4MdvcsYQQ0inI5OpvekgvAi8ABAYGoqSkBMXFxUhMTMTmzZuRlJTE7S8vL0dBQQFmzJihtLbiwoULYWVlhfnz/9el2dDQgISEBISHhyMsLKzVuktKSgAAq1evRlFREVasWAGJRIJFixZp8QoJIURPyGXqb3pIbwKviYkJhEIhnJycEBkZicjISIVloTIzMxEcHIz4+HgcPHgQ1dXV3D4+n4+NGzciOzubK5Oamorq6mq11uEdPXo01q1bh6CgIPTo0QNjxozBrFmzkJubq/0LJYSQjo5avPrJ1NQUDQ1NA5jkcjkkEgmioqLg7OwMHx8fpdWJPD09sWjRIsyaNQuHDx/G6tWrsW7dOlhaWmpU//379yEQCNp9HYQQondkMvU3PaSXgff06dPYvXs3AgKa1pctLCzE3bt3MXr0aABN93olEolSuffffx9ubm6IjIxEbGwsV55VaWkp1q1bh3feeUfziyCEEH1FLV79kJeXB0dHRwiFQowcORL+/v5cN7FEIkF4eDj4fD4AICwsDNeuXcOpU6cUjmFgYIDZs2dDJpNh3rx5Gp1HeXk5Jk6ciFGjRiE+Pr59F0UIIfqok9/j1ZvHifz9/bFmzRoYGRmhW7duMDY2BgDU1NQgNzcX9fX12Lp1K5dfKpUiIyMDvr6+CscxMmr6kxgaGjKfw+3btxESEoIBAwZgw4YN4PHoUQ1CCFGipy1ZdelN4DUzM4Orq6tS+q5du2Bra4usrCyF9JMnTyIxMRHJyckwNzdvd/2///47QkJC0L9/f3z99dcaBW5CCOkM5Hp671ZdetPV3BKxWIzQ0FB4eHgobG+++SZ4PB6ys7PbXcetW7cwbtw4ODo64rPPPkNVVRUqKipQUVEBWSf/gBFCiJJOfo9Xb1q8qhQXF+PcuXNITU1V2sfn8zF27FiIxWLExMS0q57Dhw/j2rVruHbtGjw9PRX2XbhwAY6OjkplDHhsLWJNfiGxhnxddIxLS39nLsM6BaRR4GTmOhoLs9rO9KTfy5jrgBHbe/6Yxz67jynjPTFd/PLW5KenJuflwGObBvE/qGeuw1hPZvp6LH/OPXJ6GlDVpReBd8OGDSrTvb29UVNT02K5jRs3KqUFBga2WkaVv/zlL/jLX/7CVIYQQjotaeeeMlIvAi8hhJCOQ04tXtKWlJQUrFmzRuW+oUOHYufOnTo+I0II6cAo8JK2xMXFYeLEiSr3deny7FeQIYQQvdLJB51S4FWDlZUVrKysnvdpEEKIfqAWLyGEEKJDFHgJIYQQ3ZHLKfASQgghukMtXkIIIUSHOnng1fspIwkhhLxY5DK52ltr0tLSMGDAAAiFQgQEBODYsWMt5i0sLIRAIFDaLl26pJAvJycHfn5+sLe3h5+fH/bu3auVa34StXifE2MDtinbeBpMVfciDtiXP/yDvUxpKVN+5ukfARgNi2Kro/ifzHXwrl9hyi8D+/SaUsbPCV8HnxK5jqZZZJ0LyQbGzHVUoYEpv6kOrt1QgzoanvfUl43tb/FmZ2djwYIF+Mc//oHBgwcjLS0NkZGRKCoqgrOzc4vlioqKFJ5SsbW15f7/xIkTePvtt/Hxxx8jJCQEe/fuxdSpU3Ho0CGllezaQy9avAkJCdyvF1tbW3h5eSExMREPHz5UyCcSiWBtbY38/HyF9CVLlsDT0xO1tbUK6RcuXIC9vT1ycnLUPpe6ujr8+c9/hkAgwLlz5zS/KEII0VPaaPGuW7cOkydPxpQpU+Du7o7U1FQIhUKkp6e3WrednR2EQiG3PbmS3IYNGzBs2DDMmTMH7u7umDNnDoYOHdritMSa0ovACzTNsVxSUoLi4mIkJiZi8+bNSEpK4vaXl5ejoKAAM2bMQEZGhkLZhQsXwsrKCvPnz+fSGhoakJCQgPDwcISFhal9HgsXLmz11xYhhHR67VydqL6+HsXFxQgKClJIDwoKwvHjx1utOjAwEO7u7ggNDUVBQYHCvpMnTyodMzg4uM1jstKbwGtiYgKhUAgnJydERkYiMjIS+/fv5/ZnZmYiODgY8fHxOHjwIKqrq7l9fD4fGzduRHZ2NlcmNTUV1dXVSElJUfsccnJycPz4cSxZskRr10UIIXpHxrCpUFVVBalUCjs7O4V0Ozs7VFZWqizj4OCAlStXQiwWQywWo3fv3ggLC8O//vUvLk9FRQXTMTWlt/d4TU1N0dDQdD9GLpdDIpHg008/hbOzM3x8fLBjxw7MmDGDy+/p6YlFixZh1qxZMDU1xerVq7Fr1y5YWlqqVd+NGzcwb948ZGdnw9TU9JlcEyGE6AOWRRJauxvN4ynulcvlSmnNevfujd69e3OvBw0ahBs3bmDt2rUYMmSIRsfUlN60eJ90+vRp7N69GwEBAQCaRrPdvXsXo0ePBtB0r1cikSiVe//99+Hm5obIyEjExsZy5dsilUoRFxeHDz/8UGk9XkIIIU9pZ4vXxsYGhoaGSi3RO3fuKLVYW+Pj44PffvuNey0UCtt9THXoTeDNy8uDo6MjhEIhRo4cCX9/f66bWCKRIDw8HHw+HwAQFhaGa9eu4dSpUwrHMDAwwOzZsyGTyTBv3jy1616xYgVeeuklTJ8+XXsXRAgheqq9g6v4fD68vb2VBsrm5+fDz89P7fM4f/48hEIh93rgwIHtPqY69Kar2d/fH2vWrIGRkRG6desGY+OmRwVqamqQm5uL+vp6bN26lcsvlUqRkZGhNETcyKjpT/LkSLe2FBQU4MSJEwrD0gFg+PDhiIyMxFdffaXpZRFCiP7RwlNsM2fORHx8PHx8fODn54f09HSUl5cjNjYWABAfHw8A2LhxIwBg/fr1cHFxQd++fVFfX4+srCzs379fYbDt9OnT8frrr2PlypUYP3489u3bh8LCQnz//fftP+En6E3gNTMzg6urq1L6rl27YGtri6wsxWc7T548icTERCQnJ8Pc3LxddX/11Vd49OgR9/rWrVuIjIxEenq6Vp/9IoQQfSDXQuCNiIhAdXU1UlNTUVFRgb59+yIrKwsuLi4AgLKyMoX8DQ0NSEpKwu3bt2FqasrlHzVqFJenOYAvW7YMycnJ6Nmz5zP5HtebwNsSsViM0NBQeHh4KKT36tULSUlJyM7ORkxMTLvq6NGjh8Lr5sFVPXv2hKOjY7uOTQghekdL87ZMmzYN06ZNU7nvyadaAOCDDz7ABx980OYxw8LCmB4h1YReB97i4mKcO3cOqampSvv4fD7Gjh0LsVjc7sCrCSMe28xVmtyMlzIOxDPQwWw2skeNzGXkDx62nelJv5e1necprDNRGXmPajvT03UwLv5tgFvMdehi0IapAdscUXK5rmZde/ZfZ0LwmfLfYZzpCgBMdPDvsIH3fIf3yNm/BvSKXgTelmYV8fb2Rk1NTYvlmvv+nxQYGNhqGXW4urq2+xiEEKKvtNHV3JHpReAlhBDScVDgJW1KSUnBmjVrVO4bOnQodu7cqeMzIoSQjosCL2lTXFwcJk6cqHJfly5ddHw2hBDSwWlw31+fUOBVg5WVlcIyUoQQQjRHLV5CCCFEh+QyavESQgghOkMtXkIIIUSHNHm2W59Q4CWEEKJT1OIlhBBCdEjGOq2enqHA+5wYM04ZaajBNHLsk9Wx4zGel/Qe23SDAGB0j3HKSCO2vy0A8K5fYU6f//sAACAASURBVMrPOv0jABj9aQxjiULmOljJNfhcsXYT8g3Z33OZBl2RL+JXOesUkwD7NJMafTdoeWF3Vp19cJVerMebkJAAgUAAgUAAW1tbeHl5ITExEQ8fKn5hi0QiWFtbK623uGTJEnh6eqK2tlYh/cKFC7C3t0dOTo5a5/HPf/4TI0aMgIODA1xcXBAeHt6+CyOEED0kl/HU3vSRXgReoGmO5ZKSEhQXFyMxMRGbN29GUlISt7+8vBwFBQWYMWOGwvqLALBw4UJYWVlh/vz5XFpDQwMSEhIQHh6u1koVOTk5ePfddzF58mQUFhbihx9+wOTJk7V3gYQQoifkcvU3faQ3Xc0mJiYQCoUAgMjISBQWFmL//v1YuXIlACAzMxPBwcGIj4/HwIEDUV1dDWtrawBNKxVt3LgRw4cPR0hICMaNG4fU1FRUV1cjJSWlzbobGxuxYMECLFu2DNHR0Vy6u7v7M7hSQgjp2PS1JasuvWnxPs3U1BQNDU33SuRyOSQSCaKiouDs7AwfHx/s2LFDIb+npycWLVqEWbNm4fDhw1i9ejXWrVsHS0vLNus6ffo0bt++DSMjIwwbNgzu7u6YMGECzp8//0yujRBCOjK5nKf2po/0MvCePn0au3fvRkBAAACgsLAQd+/exejRowE03euVSCRK5d5//324ubkhMjISsbGxXPm2lJaWAgA+++wzzJ07Fzt37oS9vT3GjRuHyspK7VwUIYToCblM/U0f6U3gzcvLg6OjI4RCIUaOHAl/f3+um1gikSA8PBx8ftMIw7CwMFy7dg2nTp1SOIaBgQFmz54NmUyGefPmqV237L8jXOfNm4fQ0FB4e3vjiy++gLm5Oa1cRAghT5HJeWpv+khv7vH6+/tjzZo1MDIyQrdu3WBsbAwAqKmpQW5uLurr67F161Yuv1QqRUZGBnx9fRWOY2TU9CcxNFT/kRQHBwcAwCuvvMKlGRsbo2fPnigrK9P4mgghRB/paxeyuvQm8JqZmcHV1VUpfdeuXbC1tUVWVpZC+smTJ5GYmIjk5GSYm5u3q+5XX30VxsbGuHz5MgYOHAigKbBfv34dr7/+eruOTQgh+qazD67Sm8DbErFYjNDQUHh4eCik9+rVC0lJScjOzkZMTEy76hAIBJg6dSo+//xzODg4wNnZGV999RUePHiAqKiodh2bEEL0jb4+JqQuvQ68xcXFOHfuHFJTU5X28fl8jB07FmKxuN2BF2gaWMXn8xEfH4/Hjx/Dy8sLe/fuhb29fbuPTQgh+kQq1ZvhRRrh1dTUdPLfHs/Hr31jmfJv4bP/RrJm/F1VC/ap/brJ2epwr2euApayRqb8j3ns/6hljNPuGeDZ/7MZ88tnzGXkD6qZ8stqypnrQN19puzyx4xTfgLA40fMRXZMO8GU/x77zKKwYfsoasSQ8bNlpMFHsdCE/UKWnv2avaIW3PCObjvTf7kUKz+B0tHpdYuXEELIi4e6mkmbUlJSsGbNGpX7hg4dSo8MEUIIA319TEhdFHjVEBcXh4kTJ6rc16VLFx2fDSGEdGz0OBFpk5WVFaysrJ73aRBCiF6grmZCCCFEh6irmRBCCNEh6momhBBCdKizt3g791PMhBBCdE7OsLUmLS0NAwYMgFAoREBAAI4dO9Zi3tzcXISHh8PNzQ1OTk4IDg7GgQMHFPJs27YNAoFAaXv8+LHG16oKBV5CCCE6JZUZqL21JDs7GwsWLMDs2bNRUFCAQYMGITIyEjdv3lSZ/1//+hdee+01ZGVloaCgACNHjkR0dLRSsDYzM0NJSYnCZmpqqtXrp67m54R1UJ8mHTOsS1lq8iuMuQ4dDGc01WARTynzzFXPHussVADAe8maLX89+y95XXx2NfmU9JWzzXZ1Um7GXAfr3P4aLSfL2A2rySxqPI3eFe3RxjK769atw+TJkzFlyhQAQGpqKg4fPoz09HR88sknSvlXrFih8HrBggX45z//if3798Pf359L5/F4EAqFWjjDllGLlxBCiE7JwVN7U6W+vh7FxcUICgpSSA8KCsLx48fVPo8HDx5AIBAopNXV1aFfv37w8PDApEmTcPbsWfYLbEO7A29CQoJCX7irqysmTZqES5cucXkEAgFycnK41/3798fatWtVHu/69esQCAQ4c+aMxudUX1/P9ePX1tYq7R83bhwEAoHKxROmTp0KgUCAuXPncufe2paQkIDr16/jvffeg5eXFxwcHODl5YVPP/0UdXV1Gl8DIYToK5lc/U2VqqoqSKVS2NnZKaTb2dmhsrJSrXPYtGkTfv/9d0yaNIlL6927N7788ktkZmYiLS0NJiYmGDNmDK5evarxtaqilRZvYGAg1xeenZ2Nuro6REerPwm2tu3fvx/du3eHr68vdu/erTKPk5MTtm3bBvkTXZ/V1dU4ePAgnJycuLQn+/m/+OILpbTly5fj8uXLkEqlWLlyJYqKipCSkoIdO3ZgwYIFz/ZCCSGkA5KBp/bWGh5Pcb9cLldKUyUnJweLFy/G119/DRcXFy590KBBmDx5MgYMGAB/f39888036NmzJzZu3KjZhbZAK4HXxMQEQqEQQqEQ3t7emDFjBi5duvTcWnxisRiTJk2CSCSCWCxWmWfEiBF4+PAhCgsLubSdO3fCx8cH3bt359Kar0soFMLS0lJl2ogRI7BhwwYEBwejR48eGD16NGbPno3c3Nxne6GEENIBtber2cbGBoaGhkqt2zt37ii1gp+Wk5OD6dOn46uvvsLrr7/eal5DQ0N4e3vjt99+Y7vANmj9Hu/9+/eRnZ0NDw+P5zKP8Y0bN3D06FFMmDABISEhuHz5Ms6dO6eUz9jYGCKRCBLJ/5ackkgkWlmbF2j6Ozx974AQQkjT4Cp1N1X4fD68vb2Rn5+vkJ6fnw8/P78W6/32228RHx+P9evXIywsrM3zlMvluHDhgtYHW2kl8Obl5cHR0RGOjo5wdnbGsWPHkJaWpo1DM5NIJBg+fDhsbW1hbm6O8ePHt9jqjY6Oxr59+3Dv3j2cOXMGN2/eVOvNaMvNmzexdu1avPPOO+0+FiGE6Jv2tngBYObMmcjMzERGRgZKSkowf/58lJeXIza2aa3z+Ph4xMfHc/n37NmDuLg4fPLJJ/D390dFRQUqKipw9+5dLs/y5ctx+PBhlJaW4ty5c3jvvfdw4cIFvP3221q9fq08TuTv788tm3f37l2kpaUhIiICeXl5CvdLnzWZTIbMzEz87W9/49JEIhFiY2OxdOlSpWex3N3d0a9fP+zZswfnz59HREQEzMzYHzF4UmVlJSZMmIDhw4dj5syZ7ToWIYToI208ThQREYHq6mqkpqaioqICffv2RVZWFnfPtqysTCF/eno6Ghsb8fHHH+Pjjz/m0ocMGYL9+/cDAGpra/HBBx+gsrISXbt2xYABA3DgwAH4+Pho4Yz/RyuB18zMDK6urtxrb29vuLi4YMuWLUhMTNRGFWr58ccfUVZWhri4OMTFxXHpUqkUubm5iIqKUioTHR2NtLQ0lJaWIjs7u131V1RUIDQ0FH379sXGjRvVuslPCCGdjTYCLwBMmzYN06ZNU7mvOZi29FqV5ORkJCcna+XcWvNMnuPl8XgwMDDQ+eAqsViMkJAQFBYWKmxTpkxpsbs5PDwcV69ehaOjI3x9fTWuu7y8HOPHj0efPn2wefNmGBnR3CSEEKKKlMdTe9NHWokOf/zxByoqKgAANTU12LRpEx48eIAxY8a0WOb27dtKg54cHR25/79y5QoMDQ0V9vfp06fFqbuqqqpw8OBBbNmyBR4eHgr7YmJiMHLkSFy7dg09e/ZU2GdhYYGLFy8q1cXi9u3bGD9+PBwcHJCcnIyqqipun62tbbuOTQgh+qatx4T0nVYC75EjR+Du7g6gKZD17t0bW7ZswbBhw1oss379eqxfv14hLSUlBaNHjwYAha7iZseOHVMKqs22b98OExMTBAcHK+3z9fWFo6MjxGIxFi9erLS/+TEhTf3444+4evUqrl69in79+insO3v2rMLjSc1YP3YGGnxQZYxTyWmr+6c1mvyClTJPoceOr4Orb22giCqymnLmOlingDSwfpm5DplMypRfLm1kroMnY38/HknZvs4C+MqT67TluJztu8JIg7kvTTrBKvH6f4Wt49XU1HT2v8Fz8WvfWKb8Yr4xcx3mjCHooQbBRyhn+7J7pZ65ClgwftFrEniNXsDAO+QQ+6NtPDO2R9g0Crx3VE9C3xL5w7ttZ3ra44fMRY5F5LSd6Qm2XdhvhR1vfPaB96WWpmvSoqMmbP+mAOBvZ7U3icRh9wS18waXbNBavS8KuhFJCCFEp2R6eu9WXR0q8A4ePLjFJZ9WrVqlctQyIYSQF0tn72btUIE3KysLjY2q7xe1NU0YIYSQF4MuxpO8yDpU4H1yMmtCCCEdE+u6xvqmQwVeQgghHR89TkQIIYToEN3jJYQQQnSosXM3eCnwEkII0S1q8ZLnwtCAbVyfJpNCNHT6j/ezY2rAPgGBnHEGLtTdZ6+DMT/rLFQAYGDrzFbA1hmy2sq28z3pIfusUlLG+4bXH7/EXIePMdt7choWzHXUMz7jytdgpqtnMkk/AxpcRQghzxBz0CV6r7M/TtTuHz4JCQkQCATc5urqikmTJuHSpUtcHoFAgJyc/03n1r9/f6xdu1bl8a5fvw6BQIAzZ85ofE719fVwc3ODk5MTamuVfzmPGzcOAoEAqampSvumTp0KgUCAuXPncufe2paQoDj12ePHjzFkyJB2XwMhhOgrGcOmj7TS4xAYGIiSkhKUlJQgOzsbdXV1iI6O1sahNbJ//350794dvr6+2L17t8o8Tk5O2LZtG+RPdNNUV1fj4MGDcHJy4tKar6ukpARffPGFUtry5csVjpuUlKSwyhIhhBBFcp76mz7SSuA1MTGBUCiEUCiEt7c3ZsyYgUuXLul8Pd5mYrEYkyZNgkgkanEd3hEjRuDhw4coLCzk0nbu3AkfHx+F1YSar0soFHKrGKlKA5oCfmFhIZYuXfqMrowQQjo+avFq2f3795GdnQ0PDw906dJF24dv040bN3D06FFMmDABISEhuHz5stK6vwBgbGwMkUgEiUTCpUkkEsTEsK8IAwC3bt3C7Nmz8fXXX7e4ZjAhhBAKvFoJvHl5eXB0dISjoyOcnZ1x7NgxpKWlaePQzCQSCYYPHw5bW1uYm5tj/PjxLbZ6o6OjsW/fPty7dw9nzpzBzZs3ERYWxlynVCpFXFwcZs6ciQEDBrT3EgghRK/JGTZ9pJXA6+/vj8LCQhQWFuLw4cMYNmwYIiIiUFZWpo3Dq00mkyEzMxOTJk3i0kQiEXbt2oXHj5UXCHd3d0e/fv2wZ88eiMViREREwMzMjLnef/zjHzA2NsZ7773XrvMnhJDOQMZTf9NHWnmcyMzMDK6urtxrb29vuLi4YMuWLUhMTNRGFWr58ccfUVZWhri4OMTFxXHpUqkUubm5KpcNjI6ORlpaGkpLS5Gdna1RvT/99BN+/vln2NraKqSPGDECERER2LRpk0bHJYQQfaSvXcjqeibP8fJ4PBgYGOh8cJVYLEZISAg+/vhjhfSNGzdCLBarDLzh4eH4+OOP4eLiAl9fX43qXbduHR49esS9Li8v5wKun5+fRsckhBB9xT5ti37RSuD9448/UFFRAQCoqanBpk2b8ODBA4wZM6bFMrdv31Ya9PTkYzhXrlyBoaGhwv4+ffq0OHCpqqoKBw8exJYtW+Dh4aGwLyYmBiNHjsS1a9fQs2dPhX0WFha4ePGiUl0sevToofDa3NwcANCzZ096tIgQQp6ir13I6tJK4D1y5Ajc3d0BNAWy3r17Y8uWLRg2bFiLZdavX4/169crpKWkpGD06NEAoNBV3OzYsWNKQbXZ9u3bYWJiguDgYKV9vr6+cHR0hFgsxuLFi5X2P/lIkD7RxcCEF3HwgybdWHLG6QaZp38EwDdk+50vf/yQuQ7Ws5JLG5nrkBmbMOU3sLRnrkPOYx9+0sB49Y81qONePZ8p/6vGD5jrOM9jncqy40Wxzt7VzKupqXkRvzv13mXPKUz5M4zYvuwAwJDxH6Qmczvby9l+u/WtZ64CFhrMJ8zKmPHaTXns58QaePtJxjHXwTM1ZyvAZ3/kjydwYMqvUeC9d4e5TN6fVzLlf2jA3stlJ2P7AL9k3MBcx3k5W+Dla/ANforP/oNrydmN7BW1YP2AmWrnnXFundbqfVHQXM2EEEJ0SvZC9pXpTocKvIMHD8bNmzdV7lu1apXKwVOEEEJeLJ29q7lDBd6srCw0NqruIrGzs9Px2RBCCNFE527vdrDA6+Li8rxPgRBCSDtRi5cQQgjRoc7+OJHWF0kghBBCWiODXO2tNWlpaRgwYACEQiECAgJw7NixVvMfPXoUAQEBEAqF8PLyQnp6eruPqQkKvIQQQnRKyrC1JDs7GwsWLMDs2bNRUFCAQYMGITIyssUBuKWlpYiKisKgQYNQUFCAjz76CPPmzUNOTo7Gx9QUBV5CCCE6pY0W77p16zB58mRMmTIF7u7uSE1NhVAoVNmKBYBvvvkGDg4OSE1Nhbu7O6ZMmYI333wTX375pcbH1BQFXkIIITrV3mUB6+vrUVxcjKCgIIX0oKAgHD9+XGWZEydOKOUPDg7GmTNn0NDQoNExNUWDq54TTaYcfNZe1F9hrCMgdXEdmozKlLG+548ftZ3nKayPafBkGlzJw1qm7JpM/8jratt2pqc0MNajyXvYwPjpetBgzFyHt8l9pvw/yy2Y63je3z7tHdVcVVUFqVSq9BipnZ0dKisrVZaprKxEYGCgUv7GxkZUVVVBLpczH1NT7f6OSkhIgEAg4DZXV1dMmjQJly5d4vIIBAKFfvT+/ftj7dq1Ko93/fp1CAQCnDlzRuNzqq+vh5ubG5ycnFBbq/wlMW7cOAgEAqSmpirtmzp1KgQCAebOncude2tbQkICV/bw4cMYOXIkunXrBhcXF4SGhmp8DYQQoq+0NbiKx1P8CSGXy5XS2sr/dDrrMTWhlcZBYGAgSkpKUFJSguzsbNTV1SE6Olobh9bI/v370b17d/j6+mL37t0q8zg5OWHbtm3cHx4AqqurcfDgQTg5OXFpzddVUlKCL774Qilt+fLlAIB9+/bh7bffxqRJk1BQUIAffvjhuf4NCCHkRdXermYbGxsYGhoqtUTv3LnT4mRK9vb2KvMbGRnB2tpao2NqSiuB18TEBEKhEEKhEN7e3pgxYwYuXbqk8/V4m4nFYkyaNAkikQhisVhlnhEjRuDhw4coLCzk0nbu3AkfHx90796dS2u+LqFQyK1i9HSaVCrFggUL8Le//Q3Tpk1D79694e7uTlNYEkKICjKGTRU+nw9vb2/k5+crpOfn57e4BvqgQYNw5MgRpfyvvvoqjI2NNTqmprR+O+z+/fvIzs6Gh4cHunRhX/mkvW7cuIGjR49iwoQJCAkJweXLl5XW/QUAY2NjiEQiSCQSLk0ikSAmJoa5zuLiYpSVlYHP5+O1115Dnz59EB4ejrNnz7brWgghRB/JGf5rycyZM5GZmYmMjAyUlJRg/vz5KC8vR2xsLAAgPj4e8fHxXP7Y2Fj8/vvvWLBgAUpKSpCRkYHMzEy89957ah9TW7QyuCovL49b8P3hw4dwcnJCVlaWNg7NTCKRYPjw4bC1bRqcMX78eIjFYpX3c6OjozF8+HDcu3cPV69exc2bNxEWFqYQjNVRWloKAPjss8/w2WefoXv37ti0aRPGjx+PEydOoFu3bu2+LkII0RfamDIyIiIC1dXVSE1NRUVFBfr27YusrCxuauGysjKF/D169EBWVhYWLlyI9PR0ODg4YMWKFQgLC1P7mNqilcDr7++PNWvWAADu3r2LtLQ0REREIC8vT+F+6bMmk8mQmZmJv/3tb1yaSCRCbGwsli5dClNTU4X87u7u6NevH/bs2YPz588jIiICZmZmGtULAHPmzOHexDVr1uCnn37Czp078eGHH7bjqgghRL9oa1nAadOmYdq0aSr37d+/Xylt6NChKCgo0PiY2qKVwGtmZgZXV1futbe3N1xcXLBlyxYkJiZqowq1/PjjjygrK0NcXBzi4uK4dKlUitzcXJX3XKOjo5GWlobS0lJkZ2drVK9QKATQFMibGRkZwdXVVelXFyGEdHbSTr4+0TN55JHH48HAwEDng6vEYjFCQkJQWFiosE2ZMqXFQVbh4eG4evUqHB0d4evrq1G93t7eMDExweXLl7k0mUyGa9euwdnZWaNjEkKIvmrv4KqOTist3j/++AMVFRUAgJqaGmzatAkPHjzAmDFjWixz+/ZtpUFPzfeJAeDKlSswNDRU2N+nTx+l7uJmVVVVOHjwILZs2QIPDw+FfTExMRg5ciSuXbuGnj17KuyzsLDAxYsXlepi0bVrV8TGxmL58uVwdHSEi4sLvv76a9TW1tLIZkIIeUprg6Y6A60E3iNHjnDdrBYWFujduze2bNmCYcOGtVhm/fr1WL9+vUJaSkoKRo8eDQAKXcXNjh07phRUm23fvh0mJiYIDg5W2ufr6wtHR0eIxWIsXrxYaX/zY0LtsXTpUvD5fCQkJKCurg4DBgxAbm5uiwOrbknZRnwbGbP/9qtj/L1oqkEHyB88tn9AOfzHzHWwcuCZMJdpbTJ21dj/6bA+gn9x2gnmOvrK2Wa7eiRlvw4p45U0aDBPEussVADw+i/LmPL/34A5zHXUge0Hei2PfeaqUinb59dexh7EfjaoZy6jTfraklUXr6ampnP/9HhO8t0T2s70hALTFzPwmjGWuS5/UQPvs/9nwBp++jewB0UKvOrTKPBKn33gvWvIdu0vaRB4c/jstwG/PPcNc5mWfOj1ttp5V5/V7gIFLwKaq5kQQohOdfYWb4cKvIMHD25xXcRVq1bR/VRCCOkAZPLO3dHaoQJvVlYWGhsbVe7T9lyahBBCno3OHXY7WODV9uwhhBBCdE9bE2h0VB0q8BJCCOn46HEiQgghRIcaKfASQgghukMtXkIIIUSH6HEiQgghRIfk9DgReR4aeOyz+bB69jWw/3I11OC6DRmv5D9gnw7PBuwzDD1r9zSYPvyknG1ZywB+LXMd1x+/xJT/sQazUGnSImKdiepP5/7OXEeh5wKm/PUafN7NGWeieqxBHcYavCfa1NlHNWv1r5+QkACBQACBQABbW1t4eXkhMTERDx8+VMgnEolgbW2N/Px8hfQlS5bA09MTtbWKXwYXLlyAvb09cnJy1D6X2bNnw9raWuWi9hkZGRAIBBg4cKDSvgMHDkAgEHCPLr377rvcNanabGxsAAB///vfERgYCCcnJ7i5uUEkEuHXX39V+3wJIaSz6OyrE2n9Z09gYCBKSkpQXFyMxMREbN68GUlJSdz+8vJyFBQUYMaMGcjIyFAou3DhQlhZWWH+/PlcWkNDAxISEhAeHs4tMt+Wuro67N69Gx9++GGLywGam5vjzp07KCoqUkiXSCRwcnLiXqempqKkpITbTExMFNL+/e9/AwCOHj2KuLg4/PDDD8jJyQGPx8Mbb7yBmpoatc6ZEEI6CznDf/pI64HXxMQEQqEQTk5OiIyMRGRkJPbv38/tz8zMRHBwMOLj43Hw4EFUV1dz+/h8PjZu3Ijs7GyuTGpqKqqrq5GSkqL2OeTk5KBXr1746KOP8Msvv+DSpUtKeQwNDREZGanQIq6srEReXh5EIhGXZmlpCaFQyG1A0zKAza/t7e0BAN999x3eeust9O3bF/369cNXX32FiooKnDx5Uu3zJoSQzkAGudqbPnrmHf2mpqZoaGgA0HRDXSKRICoqCs7OzvDx8cGOHTsU8nt6emLRokWYNWsWDh8+jNWrV2PdunVMS/dlZGQgKioKL730EsaNG6fUsm4WExOD7777jusK37FjB4YMGaKVGbIePHgAuVwOgUDQ7mMRQog+kcvlam/66JkG3tOnT2P37t0ICAgAABQWFuLu3bvcmrsikUjlPdj3338fbm5uiIyMRGxsLFdeHVevXsXJkycxYcIEro4dO3Zwwf9J/fv3h5ubG7799lsATd3M0dHRzNf5NLlcjvnz58Pb2xs+Pj7tPh4hhOgTuserZXl5eXB0dIRQKMTIkSPh7+/PdRNLJBKEh4eDz+cDAMLCwnDt2jWcOnVK8aQMDDB79mzIZDLMmzePqX6xWIygoCDY2toCAAICAmBsbIwDBw6ozB8TE4Nt27ahqKgIlZWVGD9+POslK5k/fz7+7//+D1u3boWBwfMdPUgIIS+azn6PV+uPE/n7+2PNmjUwMjJCt27dYGzc9JhGTU0NcnNzUV9fj61bt3L5pVIpMjIy4Ovrq3hiRk2nZmio/jMVjY2N2L59OyorK7nRxgAgk8kgFotVDs6aOHEikpKSsGzZMkRGRsLEhH0R9SfNmzcPubm52LdvH7p3796uYxFCiD6SyvW1LaserQdeMzMzuLq6KqXv2rULtra2yMrKUkg/efIkEhMTkZycDHNz83bVfejQIdy/fx8//fQTF7gB4Pr163jrrbdQVlamMGIZAAQCAcaPH49du3bhs88+07huuVyOuXPnYu/evdi3bx969eql8bEIIUSf6eugKXXprB9ULBYjNDQUHh4eCtubb74JHo+H7OzsdteRkZGBUaNGYcCAAQp1jB07Fj169MC2bdtUllu7di1+++03eHl5aVz3Rx99hKysLKSlpaFr166oqKhARUWF0jPMhBDS2VFXsw4UFxfj3LlzSE1NVdrH5/MxduxYiMVixMTEaFzH7du3kZeXh7S0NJX733jjDWzbtk3lPWNTU1OYmppqXHdjYyO++eYbAEBISIjCvkWLFmHu3LlKZcxkbF0tmnz8pIylNJnpSoPJlZ45Yw2upArKg+9aIwSfuQ5WNo3sZWSMl35crv7TAs18jO8zl7lXz/b3atCgTVDH+Glk2rxndQAAIABJREFUnYUKAIZdWM6U/wfPhcx1VBuyfS3rYoY6bZPp6WhldfFqamo691/gOTnaZzpT/rwu7G/TI0iZ8ptrEEbNGL8gb+IP5jpYp4zUpBungfFHii4C74B69veDNfD+ocG3tg+PLfCyBl1As8BrzDgGVqZByNKXwHuY/5i5zKqzmzWoSbXxHhFq5913sf29oS8amquZEEKITnX2e7wdKvCmpKRgzZo1KvcNHToUO3fu1PEZEUIIYUWBtwOJi4vDxIkTVe7r0qWLjs+GEEKIJvR1Rip1dajAa2VlBSsrq+d9GoQQQtqhs7d4aVolQgghOqXLx4n++OMPzJ07F66urnj55ZchEolw69atVsusXLkSw4cPh7OzM9zc3DBp0iRcvHhRIc+Ty+A2byNGjFDrnCjwEkII0SmpXKb21l4ff/wx9u7di82bN+PAgQO4f/8+Jk2aBKm05ac+jh49infeeQeHDh1Cbm4ujIyM8MYbb+Du3bsK+ZqXwW3edu3apdY5daiuZkIIIR2fru7x1tbWQiwWY926dRg+fDgAYOPGjejfvz+OHDmC4OBgleWentBp48aNcHFxQVFREcaOHculNy+Dy4pavIQQQnRKV+vxFhcXo6GhAUFBQVyak5MT3N3dcfz4cbWP8+DBA8hkMqVlXn/++Wf06tULPj4++Otf/4r//Oc/ah2PWryEEEJ0SldTQVZWVsLQ0FBh0RwAsLOzQ2VlpdrHWbBgAfr3749BgwZxaSNGjEBISAi6d++OGzduYNmyZQgNDcWRI0faXGyHAu9z0shjm2+Gp8EH1YBxThu2ea40K6NJFwvrzFWaMGWs4w7jFJOAbma7Yr0jZqTB999pWDDlf9X4AXMdDxqMmcvU8tjK1DP+GwTYZ6IaeeFz5jq2ey1mym/QAQcIt3fKyGXLluHvf/97q3n27t3b4j65XA6emu//woULUVRUhO+//15htbzmNd8BwNPTE97e3ujfvz8OHTqE0NDQVo+p1a7mJ0d52drawsvLC4mJiUoLBYhEIlhbWyM/P18hfcmSJfD09ERtba1C+oULF2Bvb4+cnBy1z2X27NmwtraGRCJR2peRkQGBQICBAwcq7Ttw4AAEAgFcXFwAAO+++67SyLUnt6d/SQHAihUrIBAIsGAB+1ywhBCi79o7qjkhIQEnTpxodfPx8YG9vT2kUimqqqoUyt+5cwd2dnZtnufHH3+MPXv2IDc3Fz169Gg1b7du3fDyyy/jt99+a/O4Wr/H2zzKq7i4GImJidi8eTOSkpK4/eXl5SgoKMCMGTOQkZGhUHbhwoWwsrLC/PnzubSGhgYkJCQgPDxc5Xq6qtTV1WH37t348MMPIRaLVeYxNzfHnTt3UFRUpJAukUgUlg5MTU1VGLVmYmKikPbvf/9bofzPP/+MzMxMeHh4qHWuhBDS2cjkcrU3VWxsbNCnT59WNzMzM3h7e8PY2FihkXfr1i2UlJTAz8+v1XOcP38+du/ejdzcXPTp06fNa6qqqsLt27fVGmyl9cDbPMrLyckJkZGRiIyMxP79+7n9mZmZCA4ORnx8PA4ePIjq6mpuH5/Px8aNG5Gdnc2VSU1NRXV1NVJSUtQ+h5ycHPTq1QsfffQRfvnlF1y6dEkpj6GhISIjIxVaxJWVlcjLy4NIJOLSLC0tIRQKuQ0Aunbtyr22t7fn8tbU1CA+Ph7r169H165d1T5fQgjpTHT1HK+lpSViYmKwePFiHDlyBGfPnkV8fDw8PT0RGBjI5Rs4cCC+/vpr7vWcOXOQmZmJtLQ0CAQCbpnXBw+abps8ePAAiYmJOHHiBK5fv47CwkKIRCLY2dlh/PjxbZ7XMx/VbGpqioaGpvthcrkcEokEUVFRcHZ2ho+PD3bs2KGQ39PTE4sWLcKsWbNw+PBhrF69GuvWrYOlpfrLl2VkZCAqKgovvfQSxo0bp9SybhYTE4PvvvuO6wrfsWMHhgwZwnUzs3r//fcRGRmJIUOGaFSeEEI6g/a2eFl8/vnnGD9+PGJjYzFmzBiYm5tjx44dCvdrL1++rNAdnZaWhvv37yMsLAzu7u7ctnbtWgBNDbeLFy9i8uTJ8PX1RUJCAnr16oV//vOfsLBoewzEMx1cdfr0aezevRsBAQEAgMLCQty9exejR48G0HSvd8OGDZgxY4ZCuffffx/ff/89IiMjERcXx5VXx9WrV3Hy5Eku2IpEIrz77rv45JNPYGysOPiif//+cHNzw7fffovo6GhIJBLMnz9fo8XrN2/ejFu3biE9PZ25LCGEdCa6XODe1NQUqampKteDb1ZTU9Pq66d16dJF6VlfFlpv8ebl5cHR0RFCoRAjR46Ev78/100skUgQHh4OPr9pdGdYWBiuXbuGU6dOKZ6UgQFmz54NmUymcuH61ojFYgQFBcHW1hYAEBAQAGNjYxw4cEBl/piYGGzbtg1FRUWorKxUq5vgab/++is+//xzbNq0SSm4E0IIUaTLmateRFpv8fr7+2PNmjUwMjJCt27duEBUU1OD3Nxc1NfXY+vWrVx+qVSKjIwM+Pr6Kp6YUdOpPdkd0JbGxkZs374dlZWVCqONZTIZxGKxysFZEydORFJSEpYtW4bIyMg2n79S5cSJE6iurlZ4xksqleL48ePYtGkT9ywZIYQQQK6nAVVdWg+8ZmZmcHV1VUrftWsXbG1tkZWVpZB+8uRJJCYmIjk5Gebm5u2q+9ChQ7h//z5++uknLnADwPXr1/HWW2+hrKxMYcQyAAgEAowfPx67du3CZ599plG9oaGhSj8cpk+fjldeeQUffvghBV1CCHlCZ1+dSGcTaIjFYoSGhio9ZtOrVy8kJSUhOzsbMTEx7aojIyMDo0aNwoABAxTSPTw80KNHD2zbtk3hUaVma9euxYoVK2Bt/f/t3XtcjOn/P/DXFEmRpqJsR3JOOWRZh8WyqA0dh0LOhdKuWKEth89mI6xyaLFFmomQoqR2N1myNudy2hRSSMXqtI6pfn/4mp9R0czcM1N37+fj0R9z33f365475prruq+DlkS578b0vk9NTQ1aWlo0rIgQQj7Q3NfjlctczRkZGbh69WqdTb0qKiqwtraud7xtQz169AgpKSn1jvW1s7NDVFRUnX9wVVVViQtdQggh4pHXXM2NFae0tJSd76yR+6vbPLGOT2ktfsYzMSd0VIf4TeKtxfzu9givxc5oxZK1PN6I+SEy6rX4U0xWiTn1ZSsJah7iTrX4WoIZP/sqV4j9O5lV4k1lqV4t/nt/JeZ7r5bgvbtk/k+s4w9YiDfFJACkqrwU+3c2Z4aL/Tv16WUypMHH3rx3lrHcxoLmaiaEECJXTIzPbcqaVFUiKCgI+vr6df5MnjxZ0ZdHCCGkAeQ1c1Vj1aRqvG5ubnBycqpzX+vWErTFEkIIkbvm3rmqSRW8XC4XXC5X0ZdBCCFECmztNNVQTargJYQQ0vRRjZcQQgiRo6pqmrmKEEIIkRtqaiaEEELkiJqaCSGEEDlq7uN4qeBlMSUxZzGS5L8CW57UKIt5r+SR0UKCP4iSHJrwVMT+0BT/3v5dI94sVADQQcyZqF6KOQsVIP47UZLgzyHuTFSTr4o30xUA/DlAvOVWmcbW8bkNRQUvIYQQuWruNV5GZ65asGCBcKUeHR0d9OnTB35+fnj27JnIcc7OztDS0sLJkydFtq9evRpmZmYoKysT2X7jxg106NABR48ebfC1LFmyBFpaWhAIBLX2RUZGQlNTE59//nmtfcePH4empiaMjIwAAO7u7sL3VNfP++v+FhQUYN68eejcuTP09PTwxRdf4O+//27wNRNCSHNQU1PT4B82YnzKyJEjR+LWrVvIyMiAn58fwsPD4e/vL9xfWFiI06dPw8PDA5GRkSK/6+vrCy6XK7J0X2VlJRYsWAB7e/t6Vx760IsXLxATE4NFixbVu+qRuro6njx5gvT0dJHtAoFAZM3eDRs24NatW8KfVq1aiWz7559/AAAlJSUYN24clJWVcejQIaSnp2PdunUiBTMhhBCaMpLxgrdVq1bQ1dWFgYEBeDweeDweEhMThfv37duH0aNHY968eUhKSsLTp0+F+1RUVLBz507ExsYKf2fDhg14+vQpgoKCGnwNR48eRZcuXbB48WJcv34d2dnZtY5RVlYGj8cTqREXFxcjJSUFzs7Owm3t2rWDrq6u8AcANDQ0hK87dOgAANi8eTMMDQ0RGhoKS0tLmJiYYOTIkejWrVuDr5sQQpoDqvHKmKqqKiorKwG8vdkCgQCTJk2CoaEhLC0tER0dLXK8mZkZfvjhB3h7e+PEiRMIDg7G9u3b0a5duwZnRkZGYtKkSWjTpg1sbGxq1azfcXV1xZEjR4RN4dHR0Rg6dKiwmVkciYmJsLS0xIwZM9ClSxcMGzYMYWFhrP2HQwghkmruBa9MO1ddunQJMTExGDFiBAAgLS1N2CQLvH3W+8svv8DDw0Pk97y8vJCcnAwejwc3Nzfh7zfEnTt3cOHCBWFh6+zsDHd3d6xatQotW7YUOdbc3BympqaIi4vDtGnTIBAIsGzZslrPpBsiLy8Pu3btgpeXF5YsWYKMjAwsX74cSkpKmD17dq3jh2bvFOv8Q8W+IkJIc/Bcgt/ZxODaupJ48vimQvMVjfEab0pKCvT19aGrq4sxY8ZgyJAhwmZigUAAe3t7qKi8XeDb1tYWubm5uHjxouhFKSlhyZIlqK6uho+PeN3e+Xw+Ro0aBR0dHQDAiBEj0LJlSxw/frzO411dXREVFYX09HQUFxdj/Pjx4r5lAEB1dTX69+8PPz8/WFhYYPr06Zg7dy7CwsIkOh8hhBB2YrzGO2TIEISEhKBFixbo2LGjsJZZWlqK+Ph4vH79Gnv37hUeX1VVhcjISAwYMED0wlq8vTRlZeUGZ7958wb79+9HcXGxSKem6upq8Pn8OjtnOTk5wd/fHwEBAeDxeGjVqpVY7/cdXV1d9OjRQ2Rbt27dEBERIdH5CCGEsBPjBa+amho6d+5ca/uhQ4ego6ODgwcPimy/cOEC/Pz8EBgYCHV1damyf/vtN1RUVODUqVPCght42ww8depUPHjwQKTHMgBoampi/PjxOHToENauXStx9qBBg5CTkyOy7c6dO7XyCCGENG8y71z1Dp/Px8SJE9GrVy+RHxcXF3A4HMTGxkqdERkZibFjx8LCwkIkw9raGiYmJoiKiqrz97Zu3Yq7d++iT58+Emd7enoiPT0dmzdvxt27dxEbG4uwsDC4ublJfE5CCCHsI5eCNyMjA1evXq2zqVdFRQXW1tb1jrdtqEePHiElJaXesb52dnaIioqqs5ecqqoqtLS0pMr//PPPwefzcejQIQwePBiBgYHw9/fHzJkzpTovIYQQduGUlpays782C+Tn52PlypWN+jnx7du3YWpqCk49895WVlbi7NmzYvVMJ6Q5qKmpqff/DWE3uTU1E/GVlZUhPj5e0ZfxUQMHDsSTJ0+Er3v37o38/Hzh65KSEtjb28v8OmpqavDo0SOZnNvDw0Nm537y5Am2b9+OFStWIDQ0VOReykp1dTXu378vk3O7u7vL7F4VFRVhy5YtWLp0KbZu3YqioiJGzlteXo4TJ07g0qVLtfY9f/4cmzZtYiTnQx06dMCtW7cYOde5c+dw+/Zt4euIiAgMHToUxsbGGDZsWL1zGRDFaFKLJAQFBSEkJKTOfcOGDcOBAwfkfEWNn7Gx8Se/VXM4HOTm5kp0/g+b7ktLS1FdXf3RYyTx6tUr+Pn5IT4+Hpqampg7d67I8/PHjx/DzMxMZCY0cV2/fr3O7YcOHcI333wDExMTAG+/XEjK0dERU6ZMgaOjI65fvw4bGxuoq6ujU6dOSEhIwIYNG3Ds2DGYmZlJnPHy5UusWLFC5F4tWLBAuP/Jkyfo06ePVPcqIyOjzu1xcXGwsbGBsbExAKBv374SZ9jZ2WHatGlwcnLC1atXMX78eGhoaMDU1BS//fYbNm7ciGPHjsHc3FzijFu3bsHOzg5FRUWoqalB//79sXfvXmGnyP/++w9r167FkiVLJM7w9fWtc3tVVRU2b94sfMz1008/SZzh7e2NwMBAdOnSBeHh4Vi5ciXmzp2Lbt264fbt2/Dz80NlZSXmzJkjcQZhTpNqai4pKUFJSUmd+1q3bo2OHTvK+Ypk69q1axgxYoRUH5Af+6abk5ODsLAwVFZWSlzT4nK5yM7ORvv27QEABgYGOHPmjLCQKi4uRo8ePaR6DwAQEBAAgUAAb29vlJeXIzQ0FFZWVggNDQWHw0FxcTG6d+9e77+Phr4XDodT5xeFd9s5HI5U78XY2BgpKSno2rUrnJyc0KFDB2zZsgUtWrRAdXU1li5dipycHKlaOv73v/8hOjoaS5YsQXl5ObZt24bRo0djx44dUFJSkvm9eofJe+Xg4IDPPvsMISEhUFZWRk1NDXx8fJCVlYWEhASJM9517tyxYwcqKiqwbNkyZGRkICEhAZ06dWLk3y+Xy0WvXr2goaEhsj09PR0WFhZQU1MDACQlJUmc0bFjR5w7dw5GRkb48ssv4e7uDldXV+H+2NhYrF+/HufOnZM4gzCnSdV4uVwuuFyuoi+jSZk+fXqtbcXFxQgMDASfz8fIkSOxevVq+V+YmGJiYrBlyxaMHTsWAITzgLu7u2PnzrezgEn7vKxHjx4wMjLCmjVrhOO5a2pqMHDgQERHR8PU1FS6NwHg9evXwrHp169fx4EDB4RD35SUlLBgwQKMGjVKqozY2FiEhIRgzJgxAN6OVXdychKZ0EXae9WrVy8YGBggICBA5F5ZWloiJiamziGF4qqsrBTeqxs3bmDVqlXC1xwOB/PmzZP6Xl24cAHx8fHQ0NCAhoYGBAIBfHx8YGNjg2PHjqFNmzZSv4/ly5dj//792LhxIwYPHizcrqOjg507d9Ya/y8JNTU1PH36FEZGRigsLES/fv1E9vfr10/kERBRLHrGq0DOzs4f/Vm+fDmjee+azfr374/MzEwcPnwYMTExUjWdcjgclJaWClsjPnwtTa3qfUVFRejevbvwtYmJCRISEpCRkQE3Nze8efNG6oyTJ09CX18fc+bMwatXr9CpUydhAWJgYIBOnTqhU6dOUmX07t0baWlpAN4+43vw4IHI/vv376N169ZSZRQVFYkszmFoaIiEhATcvHkTs2fPFs6dLo3U1FQYGRlhxowZqKiogJGRkbB5WU9PD0ZGRhLNef6+3r1749SpU8Jzflhw5OfnC2uLknr16lWtSXqCgoIwfvx42NjY1BqbL4lly5bhl19+wfz587F27dpaj2KYMGbMGISHv50GcujQobVaTI4ePcrIlyHCjCZV42WbTw1h0tLSEn6YSePNmzfYvXs3goKC0LZtW4SEhMDR0VHq8wJvazmDBg0Sef1+LYSpnpsdOnRAbm6uyP3Q09NDfHw8JkyYUGu+b0moqqri559/xtGjR2FnZ4fFixfD3d1d6vO+b+nSpZg3bx5UVFTg6ekJX19flJWVoXv37sjJycHatWsxadIkqTJ0dXVr3at361lPmDAB8+fPl/ZtCJfHTExMhJOTE7y8vBj5G7xv2bJlmDNnDlq0aIEFCxbghx9+QElJifBerVu3TmQlMUl06dIFV65cEflSB7wtfKurqzFlyhSpzv/O4MGDcerUKXz77bcYO3Ysfv31V0bO+86qVaswbtw4jB8/HpaWlti6dSvOnj0rvFfp6elSD9kkzGlSz3iJ+OLi4vDjjz+ioqICS5cuxaxZs2otFiGNM2fONOi4YcOGSZWzcOFCcDgcbN26tda+goIC2NjYIC8vT+pnye/k5+fD3d0dbdu2xcmTJ3HmzBlGmgSBt7WPFStWoLCwUPiMlMPhoEWLFpgxYwYCAwNFZl4Tl5eXF6qrq7F9+/Za+woLC2FjY4Pc3FzG7tXDhw/h7u4OFRUVpKWlMXqvjh07huXLl6OgoEDkeXKrVq0wa9YsBAQEiDWt7Ic2btyIs2fP1juBj7e3NyIiIhhruQGA8PBwBAYGoqSkBH/99Rdj96q0tBQ///wzjh8/jry8PFRXV0NXVxeDBg2Cp6dnrWl5ieJQwduIpaWlgc/nY9euXRKfg8vlonXr1nB0dETbtm3rPU6aHpXycO/ePWRnZwuf8X6ooKAAKSkpdT7TllR1dTXWrVuHtLQ07Ny5U+qm0/dVVVXh0qVLuHfvHqqrq6Gnp4d+/fqJtfxlffLz85GTk4PRo0fXub+wsBCpqamM1eaAt/dq48aNOH36NEJDQxm9V9XV1cjIyBDeK11dXfTt2/ej/56ZVFVVJVXhXpesrCycP38etra2jPzNSdNCBW8jU1RUhKioKAgEAuTl5eGLL75AYmKixOezsrJqUFOvND0q35eeno6qqirha2VlZXzxxReMnFscLi4uCA4Ohq6ubrPL8PT0xLp162RaMLElo3379vjnn3+Eq5lRBpEHKngbgerqaiQnJ4PP5yMlJQVVVVVYtmwZZs6cKdMPdSacOnUKK1aswNmzZwG87YT0/PlzkSbU6OjoemuqsvLhsKbmlKGlpYVbt24Jh3jJAlsyPhwORxlEHqhXswLdvXsXq1evRs+ePeHr6wsLCwukp6dDSUkJtra2Mil0S0tLUVZWxtj59uzZg1mzZolsO3HiBO7cuYPbt29j0aJFIstAEtljYsKS5pJBiCJQr2YFGjRoEJydnRERESEyvo9pFRUV+Omnn3D48GHhRBna2trg8XhYsWKFVE15GRkZWLhwocg2Lpcr7LFtZ2cHHo8n+cUTichjDmC2ZCQkJHzy/4C0/4bZkkGYQQWvAnXt2hUpKSnQ1NRE27ZtpRpPW5+ysjJYWVnh3r17cHBwQI8ePVBTU4OsrCzs2bMHf/75J5KTk2vNqtNQjx49QocOHYSvw8PDRWrqmpqajPYIJQ3z/jje+kjbq5ktGfVN6fgOh8ORusBiSwZhBhW8CnT27FlcvnwZfD4f33zzDQwMDITjEpn6pr9x40ZUVVXh4sWL0NfXF9nn6+sLBwcHbNq0CWvWrJHo/G3btkVeXp6wF+u4ceNE9t+7d0/iQp1ILjg4WOa9ZdmSce3aNZk/G2VLBmEGFbwK1r9/f/Tv31/YFBwVFYWqqip4eXmBx+NhwoQJ0NPTk/j8CQkJWL9+fa1CF3jbcWfNmjVYsWKFxAXv559/jv379+PLL7+sc79AIKDxgwpgbW0t8w9htmQQIm/UuaqRaN26NaZNm4akpCScP38egwYNwoYNG6RapQZ42xT8sSZsCwsLqZZx8/T0xIEDB+Dv74/Hjx8LtxcXF2P58uU4fPhwrWfAkhB3IvzvvvsOmpqazTKDLc9eaa1awlZU8DZCXbt2xY8//oibN28iIiJCqnO1a9fuo+uWPnr0SKqm4GHDhmHjxo3YtWsXunfvDmNjY5iYmKBHjx7CaSqlnbUKAObOnQsPDw9UVFQ06PilS5eKXWCxJYMtPY7lkcHj8aCqqkoZRK5oHG8jdPr0abx48QKDBg0S+0P3Q+8m/BcIBHXunzp1Klq1aoXdu3dLlfPw4UMcOXIEd+/eBQCYmprC1tYW+vr6+PPPPzFy5Eipzn/jxg14eHjg6dOnCA0NrbdpmzIIIY0dFbwK9N9//2H16tXIzMzEgAED4O/vDx6PJ5yMQkdHB0eOHJGqufnWrVsYPXo0evbsiYULF6Jbt27gcDjIysrCtm3bkJWVhZSUFMbmi32noKAAUVFR4PP5ePDgASPzAr958wZBQUEICQnBnDlzsHTp0lpT+UnbkYsNGT4+Pg06LigoqNlnNHRiknv37jX7DMIcKngVaPHixfj9999hb2+PEydOgMvl4vXr1wgMDISSkhJWrlwJdXV1HDhwQKqcCxcuwNPTEzk5OcLnZjU1NejSpQu2bdsmsrqQNKqqqnD8+HHw+XykpqbCzMwMDg4OsLW1ZXR2p9TUVPB4vDqbIpma+L8pZ3C5XBgYGMDExKTe5loOhyPVAvJsyjA0NISzszMMDAzqPU6aOcDZkkGYQwWvAvXq1Qu//PILRowYgfv378PCwgLx8fHC5sdLly7BxcUF2dnZUmfV1NTgypUruHPnDoC3y6H17duXkQ4sOTk5iIyMRHR0NNTU1MDj8RAcHMzoKjXvJCUlwdvbG506dcL3339faxWfESNGNPuMmTNnIjk5GQMGDICrqytsbW0Zf/7HloykpCQIBAKkpKRg2LBhmDZtGsaPH8/oCl5sySDMoYJXgbS1tXHjxg3hcKGOHTvi7NmzwsXWi4qK0LNnT8ZqWLJgbW2NmzdvwtbWFpMmTRJ2pNLR0WG04K2oqICPjw/i4uLg6+sLLy8vxnu9siUDAEpKSnDgwAEIBALcv38fjo6OcHV1Rb9+/SijDo8fP0Z0dDQEAgEeP34MHo8HV1dXRie1YUsGkR4VvAr04cTmH06IX1xcjB49ekhV8O7YsaNBx0m6OLq2tjbmzp2LGTNmoFevXsLtTBe8vXv3hpaWFnbs2CGSwyS2ZHzoypUrEAgEOHz4MPT19fHHH39ATU2NMupx7tw5BAQE4K+//sLdu3el7uDI5gwiGZpAQ8F2794NdXV1AG873fD5fHC5XABvO19Jq66F4z/E4XAkLnhPnjyJyMhIWFtbw8jICM7OznBycpLoXB/D4/Hg6+sr06YztmR8qGvXrjA3N8fFixdx584dkWUbKeP/e/HiBeLi4sDn85GZmQknJyfGC3a2ZBDpUI1XgczNzRvUzHj16lU5XI10Xr58iSNHjoDP5+PcuXOorq7G6tWrMX36dJl/037x4gViY2PB5/ORnJxMGf/nzJkz4PP5OHbsGMzNzTFlyhQ4ODigTZs2DFwtezLOnz8PPp+PI0eOwNTUFK6urnBycmJ0qkq2ZBBmUI1Xga5duybzDHt7e0RERMj8P5+qqiqcnZ3h7OyMu3fvIjIyEqGhoQgICMDw4cMRExNJhOekAAAOi0lEQVTDeObly5cRGRmJ2NhYcDgcWFtbUwaATZs2ISoqCs+ePYOzszNOnjzZoMUGmmPGoEGD8OTJE/B4PCQnJ0s9UxybMwhzqMbbhAwZMgQHDx786HCBDylygeyqqiokJydDIBBg//79jJyztLQU0dHR4PP5yM3NxcuXLxEcHAwXFxfGmm+besa7YThjxoyp1Vv6fdKMf2VThrq6OpSVlT/a+iTN+Fe2ZBDmUI23CcnPz8ebN28UfRkNpqysDBsbG9jY2Eh9rlOnTmHv3r1ISkqCpaUlFixYgIkTJ6Jz584YOHAgIwUiWzKGDBkCDofDyDA0tmeEhITI7NxsyyDMoYK3GSgqKvpkgd2xY0c5XY1kHBwc4OnpiQsXLohV42+OGYmJiTI5Lxsz5DGhBFsyCHNokYRmYPjw4TAzM6vzp1evXk3iedCYMWMQHh6OH374AcePH5dJr1m2ZHzKvXv3EBAQQBmfUFBQgA0bNqBv376UQRhFNd5m4MCBA8IhSk1VdHQ0ioqKsG/fPvj7+8PLywv29vYAmFs+ji0ZdXn9+jXi4+PB5/ORlpYGAwMD+Pn5UcYHPpz21NjYGI6Ojoydn00ZRHLUuaoJ+XCCjYZQZOcqWTp9+jQEAgESEhKgo6MDW1tb2NnZYcCAAZTxnuvXr4PP5+PgwYMoKyvD9OnT4erqCktLS0bOz5aM27dvC6c9bdGiBYqKihAVFQUrKytGzs+mDCI9ampmObYuJj58+HDs2rULWVlZ+Pbbb5GWloaxY8dSBt5OS7lnzx589dVXsLKyQnl5OXbv3g0lJSXMnz+fkcKKLRn79u2DlZUVRo4cicLCQuzcuRPXrl0Dh8NhbGEPtmQQ5lBTswJNmDABfD6/wRNMbN68Weyaa8eOHWstOdcUPX/+HCtXrkRiYiIqKysxcuRIrF+/Htra2nBzc4ObmxsyMzMpA4CZmRkGDBiA+fPnY8KECTKZtYgtGV5eXvD29kZsbKzMZndiSwZhDtV4FejMmTOorKxs8PE8Hk84vWRD3bhxA1paWnXue/HiRZNphgoMDMS+ffswduxYODo64uTJk1i8eLHIMX369KEMAC1btkR5eTnKy8vx6tUrqc7F9oyZM2ciLCwMtra2CAsLQ0lJCWUQmaOCtxm6fPkyFi1ahO7du8PX17dJNEUlJCRg69atCAkJwfr163Hw4EEkJiYy2iuYLRlZWVlYuHAhkpOT0bNnTzg7O+PIkSOMPnZgS8amTZuQlZWFOXPmIC4uDj169ICLiwtqamoY+5uwJYMwhzpXKRCXy8XJkyehra390eMMDQ2lzpLHbEyy1L59e2RmZuKzzz4TbtPT08PFixcZGw/Lloz35efnIyoqCvv27cODBw8wceJETJkyBaNGjWLs786WDAC4c+eOcKa1//77D2PHjoWtrS1sbW0pgzCGCl4F4nK5H/32XlNTAw6HI9WygB/OlOTs7CycKUkWC9XLipaWFrKzs6GjoyPcJkkv7+aQUZeamhqkpqaCz+cjKSkJrVq1Qn5+PmXU4910p5GRkUhNTcXjx48pgzCGCl4F4nK5OHz4cL3PYN+RZuC7trY2PD094e7uLlKjYnq9XFnjcrn46quvoKKiItyWkpKCoUOHonXr1sJt0dHRzT7jU/79919ER0fD09MTABATEwNra2ux+w80hwwAKCwshJ6eHgDAxcUFwcHB0NXVpQwiMSp4FUgeY2ydnZ2RlpaGr7/+GpMnT8a4ceOgrKzc5ApeDw+PBh0XGhra7DPEZWhoiLS0NJnWuNmSIY/WCbZkkPrRcCKWU9RMSUyTR0HElgxx1dTI/rs3WzIIYQL1alYgQ0NDuYyx1dXVhbe3Ny5duoQ9e/agvLwcLVu2hJOTE/z8/HDx4kWZXwMhhJC3qMarQFevXpV75vDhwzF8+HCUlZXh4MGDEAgECA0NlaoDFyGEkIajgleBnJ2dG3ScLDratGvXjrGZkgghhDQcNTUrkJaWVoN+pHHz5k1MnjwZ5eXltfaVlZVh8uTJUFVVlSqDEEJIw1GNV4Hk0dFm27Zt6N27NzQ0NGrta9euHSwsLLBlyxZs375d5tdCGodXr14hNzdX2KPd0NAQLVow+1HQVDISEhIwYcKEBh//3XffNXhudbZlEObQcCKWs7S0RHh4eL1jgTMzMzFr1ixcvnxZzldGFOXatWsYMWKETJ/rN5UMXV1dODo6Yv369Wjbti2DV8e+DMIcqvEqkI+PT4OOCwoKkjjj4cOHH22u5nK5KCgokPj8hDRlqamp8PDwwJAhQxAaGoovv/ySMojMUcGrQL/++isMDAxgYmJS7xhEacfatmvXDrm5uTAyMqpz/927d9GuXTupMghpqszMzHDixAkEBQXByckJc+bMwdKlS2sN86vrUU1zyyDMoaZmBZo5cyaSk5MxYMAAuLq6wtbWlvGOTrNnz8bz58/r7Rk9efJkqKmpYc+ePYzmksarqTQDyzsjNTUVPB6vzi/BlEGYRDVeBYqIiEBJSQkOHDiArVu3wsfHB46OjnB1dUW/fv0YyVi0aBHGjBmDqVOnwtvbG127dgUAZGdnY/Pmzfjzzz/x+++/M5JFGoeMjIyP7r99+zZlfCApKQne3t4YOHAgvv/+e8Y7grEpg0iParyNyJUrVyAQCHD48GHo6+vjjz/+gJqamtTnTU5OxsKFC2t929XS0sKWLVvwzTffSJ1BGo93q159bApFaVe9YktGRUUFfHx8EBcXB19fX3h5eTE+lSpbMghz6OtQI9K1a1eYm5vj4sWLuHPnDmMLWFtZWeHatWtISUlBbm4uampqYGpqilGjRjFSsJPGRR4TorAlY/DgwdDS0kJqaip69epFGUQuqMbbCJw5cwZ8Ph/Hjh2Dubk5pkyZAgcHB7Rp00bRl0ZY6OnTp4iNjcXcuXObfcaaNWvg6+uLli1bMnhl7MwgzKGCV4E2bdqEqKgoPHv2DM7Ozpg6dSq6devGaEZ9Q5Y0NDTQtWtX2NnZoVWrVoxmksatKXZ8UkTGixcvEBsbCz6fj+TkZMogjKGmZgUKCAiAgYEBbGxs8OLFC4SFhdV5nDTjeG/evFnn9rKyMuzYsQOBgYFISEiAoaGhxBmEsMnly5cRGRmJ2NhYcDgcWFtbUwZhFBW8CjRkyBBwOBxkZ2fXe4y0HSSOHTtW777y8nK4ublhzZo19Rb6hDQHpaWliI6OBp/PR25uLl6+fIng4GC4uLgw1nzLlgwiPSp4FSgxMVGh+RoaGvDx8cGMGTMUeh2EKMqpU6ewd+9eJCUlwdLSEgsWLMDEiRPRuXNnDBw4kJHCii0ZhDlU8DZz2traKCsrU/RlEAZt27bto/sLCwsp4/84ODjA09MTFy5cgIGBgdTnY3MGYQ4VvAr0qQ+WdxYuXCizazh//jyMjY1ldn4if7t27frkMdJ+OLMlY8yYMQgPD0deXh4mT56McePG1ZpmUVpsySDMoV7NCmRhYfHJYzgcjlTjGeub/ae8vByZmZnYvHkzVqxYATc3N4kzCGnKioqKsG/fPggEApSWlsLe3h4RERH466+/0L17d8ogjKOCtxHLy8vDqlWrEBERIfE5Pjb7j7a2Njw9PbFo0SKa5YYQAKdPn4ZAIEBCQgJ0dHRga2sLOzs7DBgwgDIIY6jgbcSYGKeYn59f53YNDQ1aCJul4uPjG3TcxIkTm31GfcrKynDw4EEIBAJcu3ZNJmOF2ZJBxEcFbyMmj0kICPtwudxPHsPEPMpsyHj+/DlWrlyJxMREVFZWYuTIkVi/fj20tbWFx2RmZqJPnz7NPoMwhwreRozJgvfMmTOIj49HXl4eAMDExAR2dnYYPHiw1OcmpKny9/dHeHg4eDweVFVVERMTg2HDhmHv3r2UQWSGCt5GjKmCd9myZdi1axc0NTXRpUsX1NTU4Pbt2ygvL4e7uzvWrVvH0BWTxuTVq1fC6UALCgoQERGBly9fwsrKCkOGDKEMAH379oW/vz8cHR0BAJcuXcK4ceNQVFTEWK9gtmQQ5lDBq0DOzs4f3V9RUYG///5bqoI3OTkZ06ZNw6ZNm+Dq6golJSUAQHV1Nfbu3QsfHx8IBAKMGzdO4gzSuOTk5MDV1RXZ2dkwMzPDr7/+Cnt7e1RUVEBJSQnPnj3D3r17MX78+Gaf0b59e2RmZuKzzz4TbtPT08PFixcZGw/LlgzCHCVFX0BzpqWl9dEfY2PjTxbOnyIQCDBv3jzMmDFDWOgCgJKSEmbNmoW5c+eCz+dL+1ZII+Ln5wddXV3s378fPXv2BI/Hw6hRo5Cfn4979+5h1qxZCA4OpgwAVVVVUFFREdnWokULvHnzRqrzsjGDMIdqvCzXu3dv7N69GwMHDqxz//nz5zF79mxcv35dzldGZMXU1BRxcXGwsLBARUUFjIyMkJqain79+gEAsrOz8fXXX9fb4705ZXC5XHz11VcihVZKSgqGDh2K1q1bC7dFR0c3+wzCHJq5iuWePHkCfX39evfr6+vjyZMncrwiImtPnz5Fx44dAQBt27aFurq6SA9hTU1N/Pfff5QBwMXFpda2SZMmSXVOtmYQ5lDBy3KvXr2q1QT1vpYtW+L169dyvCIiD/KYEIUNGaGhoTI9P5syCHOo4G0Gdu/eDXV19Tr3SVtjII2Tu7u78AvXy5cv8d133wmbHJn6osWWDELkjZ7xspy5uXmDag1Xr16Vw9UQefDw8GjQcdLUktiSQYgiUMFLCCGEyBENJ2K5sWPHorS0VPh6zZo1KCkpEb7+999/0bt3b0VcGiGENEtU8LLchQsXUFlZKXwdFhYmsvB9VVUVHj58qIhLI4SQZokK3mamruUBaUlAQgiRHyp4CSGEEDmigpflOBwO1WgJIaQRoXG8LFdTU0NjIQkhpBGh4UQsR2MhCSGkcaGClxBCCJEjesZLCCGEyBEVvIQQQogcUcFLCCGEyBEVvIQQQogcUcFLCCGEyNH/A0zF3Nmn4cYTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.style as style\n",
    "\n",
    "style.use('fivethirtyeight')\n",
    "sns.heatmap(credit_card_default.corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LIMIT_BAL                    -0.153520\n",
       "SEX                          -0.039961\n",
       "EDUCATION                     0.028006\n",
       "MARRIAGE                     -0.024339\n",
       "AGE                           0.013890\n",
       "PAY_0                         0.324794\n",
       "PAY_2                         0.263551\n",
       "PAY_3                         0.235253\n",
       "PAY_4                         0.216614\n",
       "PAY_5                         0.204149\n",
       "PAY_6                         0.186866\n",
       "BILL_AMT1                    -0.019644\n",
       "BILL_AMT2                    -0.014193\n",
       "BILL_AMT3                    -0.014076\n",
       "BILL_AMT4                    -0.010156\n",
       "BILL_AMT5                    -0.006760\n",
       "BILL_AMT6                    -0.005372\n",
       "PAY_AMT1                     -0.072929\n",
       "PAY_AMT2                     -0.058579\n",
       "PAY_AMT3                     -0.056250\n",
       "PAY_AMT4                     -0.056827\n",
       "PAY_AMT5                     -0.055124\n",
       "PAY_AMT6                     -0.053183\n",
       "default payment next month    1.000000\n",
       "Name: default payment next month, dtype: float64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# just correlations between every feature and the response\n",
    "credit_card_default.corr()['default payment next month']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LIMIT_BAL                     False\n",
       "SEX                           False\n",
       "EDUCATION                     False\n",
       "MARRIAGE                      False\n",
       "AGE                           False\n",
       "PAY_0                          True\n",
       "PAY_2                          True\n",
       "PAY_3                          True\n",
       "PAY_4                          True\n",
       "PAY_5                          True\n",
       "PAY_6                         False\n",
       "BILL_AMT1                     False\n",
       "BILL_AMT2                     False\n",
       "BILL_AMT3                     False\n",
       "BILL_AMT4                     False\n",
       "BILL_AMT5                     False\n",
       "BILL_AMT6                     False\n",
       "PAY_AMT1                      False\n",
       "PAY_AMT2                      False\n",
       "PAY_AMT3                      False\n",
       "PAY_AMT4                      False\n",
       "PAY_AMT5                      False\n",
       "PAY_AMT6                      False\n",
       "default payment next month     True\n",
       "Name: default payment next month, dtype: bool"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter only correlations stronger than .2 in either direction (positive or negative)\n",
    "credit_card_default.corr()['default payment next month'].abs() > .2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PAY_0', 'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5',\n",
       "       'default payment next month'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# store the features\n",
    "highly_correlated_features = credit_card_default.columns[credit_card_default.corr()['default payment next month'].abs() > .2]\n",
    "\n",
    "highly_correlated_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PAY_0', 'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5'], dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the response variable\n",
    "highly_correlated_features = highly_correlated_features.drop('default payment next month')\n",
    "\n",
    "highly_correlated_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8196666666666667\n",
      "Best Parameters: {'max_depth': 3}\n",
      "Average Time to Fit (s): 0.01\n",
      "Average Time to Score (s): 0.002\n"
     ]
    }
   ],
   "source": [
    "X_subsetted = X[highly_correlated_features]\n",
    "\n",
    "\n",
    "# barely worse, but about 20x faster to fit the model\n",
    "get_best_model_and_accuracy(d_tree, tree_params, X_subsetted, y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "\n",
    "class CustomCorrelationChooser(TransformerMixin, BaseEstimator):\n",
    "    def __init__(self, response, cols_to_keep=[], threshold=None):\n",
    "        # store the response series\n",
    "        self.response = response\n",
    "        # store the threshold that we wish to keep\n",
    "        self.threshold = threshold\n",
    "        # initialize a variable that will eventually\n",
    "        # hold the names of the features that we wish to keep\n",
    "        self.cols_to_keep = cols_to_keep\n",
    "        \n",
    "    def transform(self, X):\n",
    "        # the transform method simply selects the appropiate\n",
    "        # columns from the original dataset\n",
    "        return X[self.cols_to_keep]\n",
    "        \n",
    "    def fit(self, X, *_):\n",
    "        # create a new dataframe that holds both features and response\n",
    "        df = pd.concat([X, self.response], axis=1)\n",
    "        # store names of columns that meet correlation threshold\n",
    "        self.cols_to_keep = df.columns[df.corr()[df.columns[-1]].abs() > self.threshold]\n",
    "        # only keep columns in X, for example, will remove response variable\n",
    "        self.cols_to_keep = [c for c in self.cols_to_keep if c in X.columns]\n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate our new feature selector\n",
    "ccc = CustomCorrelationChooser(threshold=.2, response=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PAY_0', 'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ccc.fit(X)\n",
    "\n",
    "ccc.cols_to_keep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>PAY_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PAY_0  PAY_2  PAY_3  PAY_4  PAY_5\n",
       "0      2      2     -1     -1     -2\n",
       "1     -1      2      0      0      0\n",
       "2      0      0      0      0      0\n",
       "3      0      0      0      0      0\n",
       "4     -1      0     -1      0      0"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ccc.transform(X).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 300)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# recall this is used to construct features to model polynomial interactions\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "# instantiate our constructor\n",
    "poly = PolynomialFeatures(degree=2)\n",
    "\n",
    "poly_X = poly.fit_transform(X)\n",
    "\n",
    "poly_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_pipe_params = {'classifier__max_depth': \n",
    "                    [None, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classifier__max_depth': [None, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21], 'poly_features__degree': [2]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8199666666666666\n",
      "Best Parameters: {'classifier__max_depth': 3, 'poly_features__degree': 2}\n",
      "Average Time to Fit (s): 3.492\n",
      "Average Time to Score (s): 0.031\n"
     ]
    }
   ],
   "source": [
    "# construct a pipeline using a decision tree and polynomial feature construction\n",
    "from copy import deepcopy\n",
    "poly_pipe_params = deepcopy(tree_pipe_params)\n",
    "poly_pipe_params.update({'poly_features__degree':[2]})\n",
    "\n",
    "print(poly_pipe_params)\n",
    "\n",
    "pipe = Pipeline([('poly_features', poly),\n",
    "                 ('classifier', d_tree)])\n",
    "\n",
    "get_best_model_and_accuracy(pipe, poly_pipe_params, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classifier__max_depth': [None, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21], 'correlation_select__threshold': [0, 0.1, 0.2, 0.3]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8206\n",
      "Best Parameters: {'classifier__max_depth': 5, 'correlation_select__threshold': 0.1}\n",
      "Average Time to Fit (s): 0.091\n",
      "Average Time to Score (s): 0.003\n"
     ]
    }
   ],
   "source": [
    "from copy import deepcopy\n",
    "\n",
    "# instantiate our feature selector with the response variable set\n",
    "ccc = CustomCorrelationChooser(response=y)\n",
    "\n",
    "# make our new pipeline, including the selector\n",
    "ccc_pipe = Pipeline([('correlation_select', ccc), \n",
    "                     ('classifier', d_tree)])\n",
    "\n",
    "# make a copy of the decisino tree pipeline parameters\n",
    "ccc_pipe_params = deepcopy(tree_pipe_params)\n",
    "\n",
    "# update that dictionary with feature selector specific parameters\n",
    "ccc_pipe_params.update({\n",
    "               'correlation_select__threshold':[0, .1, .2, .3]})\n",
    "\n",
    "print(ccc_pipe_params)\n",
    "\n",
    "# better than original (by a little, and a bit faster on \n",
    "# average overall\n",
    "get_best_model_and_accuracy(ccc_pipe, ccc_pipe_params, X, y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LIMIT_BAL', 'PAY_0', 'PAY_2', 'PAY_3', 'PAY_4', 'PAY_5', 'PAY_6']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the threshold of .1\n",
    "ccc = CustomCorrelationChooser(threshold=0.1, response=y)\n",
    "ccc.fit(X)\n",
    "\n",
    "# kept a few columns\n",
    "ccc.cols_to_keep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 使用假设检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.30224395e-157 4.39524880e-012 1.22503803e-006 2.48536389e-005\n",
      " 1.61368459e-002 0.00000000e+000 0.00000000e+000 0.00000000e+000\n",
      " 1.89929659e-315 1.12660795e-279 7.29674048e-234 6.67329549e-004\n",
      " 1.39573624e-002 1.47699827e-002 7.85556416e-002 2.41634443e-001\n",
      " 3.52122521e-001 1.14648761e-036 3.16665676e-024 1.84177029e-022\n",
      " 6.83094160e-023 1.24134477e-021 3.03358907e-020]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>PAY_0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>PAY_2</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>PAY_3</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>PAY_4</td>\n",
       "      <td>1.899297e-315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>PAY_5</td>\n",
       "      <td>1.126608e-279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  column        p_value\n",
       "5  PAY_0   0.000000e+00\n",
       "6  PAY_2   0.000000e+00\n",
       "7  PAY_3   0.000000e+00\n",
       "8  PAY_4  1.899297e-315\n",
       "9  PAY_5  1.126608e-279"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "import pandas as pd\n",
    "\n",
    "k_best = SelectKBest(f_classif, k=5)\n",
    "k_best.fit_transform(X, y)\n",
    "print(k_best.pvalues_)\n",
    "p_values = pd.DataFrame({'column': X.columns, 'p_value': k_best.pvalues_}).sort_values('p_value')\n",
    "\n",
    "p_values.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>BILL_AMT4</td>\n",
       "      <td>0.078556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>BILL_AMT5</td>\n",
       "      <td>0.241634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>BILL_AMT6</td>\n",
       "      <td>0.352123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       column   p_value\n",
       "14  BILL_AMT4  0.078556\n",
       "15  BILL_AMT5  0.241634\n",
       "16  BILL_AMT6  0.352123"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_values[p_values['p_value'] >= 0.05]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Pipeline' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-c6a45d624355>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# Make a new pipeline with SelectKBest\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m select_k_pipe = Pipeline([('k_best', k_best), \n\u001b[0m\u001b[1;32m      5\u001b[0m                           ('classifier', d_tree)])\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Pipeline' is not defined"
     ]
    }
   ],
   "source": [
    "k_best = SelectKBest(f_classif)\n",
    "\n",
    "# Make a new pipeline with SelectKBest\n",
    "select_k_pipe = Pipeline([('k_best', k_best), \n",
    "                          ('classifier', d_tree)])\n",
    "\n",
    "select_k_best_pipe_params = deepcopy(tree_pipe_params)\n",
    "\n",
    "select_k_best_pipe_params.update({'k_best__k':range(1,23)+['all'],  # the 'all' literally does nothing to subset\n",
    "                                 })\n",
    "print(select_k_best_pipe_params)\n",
    "# comparable to our results with correlationchooser\n",
    "get_best_model_and_accuracy(select_k_pipe, select_k_best_pipe_params, X, y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv('~/开发/AI相关/特征工程/Feature-Engineering-Made-Easy-master/data/twitter_sentiment.csv', \n",
    "                     encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItemID</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>SentimentText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>is so sad for my APL frie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>I missed the New Moon trail...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>omg its already 7:30 :O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>.. Omgaga. Im sooo  im gunna CRy. I'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>i think mi bf is cheating on me!!!   ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ItemID  Sentiment                                      SentimentText\n",
       "0       1          0                       is so sad for my APL frie...\n",
       "1       2          0                     I missed the New Moon trail...\n",
       "2       3          1                            omg its already 7:30 :O\n",
       "3       4          0            .. Omgaga. Im sooo  im gunna CRy. I'...\n",
       "4       5          0           i think mi bf is cheating on me!!!   ..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_X, tweets_y = tweets['SentimentText'], tweets['Sentiment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    0.564632\n",
       "0    0.435368\n",
       "Name: Sentiment, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# null accuracy\n",
    "tweets_y.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: After pruning, no terms remain. Try a lower min_df or a higher max_df.\n",
      "\n",
      "  FitFailedWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.7557531328446129\n",
      "Best Parameters: {'featurizer__max_df': 0.7, 'featurizer__max_features': 10000, 'featurizer__min_df': 0.0, 'featurizer__ngram_range': (1, 2)}\n",
      "Average Time to Fit (s): 2.796\n",
      "Average Time to Score (s): 0.44\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# import a naive bayes to help predict and fit a bit faster \n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "featurizer = CountVectorizer()\n",
    "\n",
    "text_pipe = Pipeline([('featurizer', featurizer), \n",
    "                 ('classify', MultinomialNB())])\n",
    "\n",
    "text_pipe_params = {'featurizer__ngram_range':[(1, 2)], \n",
    "               'featurizer__max_features': [5000, 10000],\n",
    "               'featurizer__min_df': [0., .1, .2, .3],   \n",
    "               'featurizer__max_df': [.7, .8, .9, 1.]}\n",
    "\n",
    "\n",
    "get_best_model_and_accuracy(text_pipe, text_pipe_params, \n",
    "                            tweets_X, tweets_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.7529728270109712\n",
      "Best Parameters: {'select_k__k': 5000}\n",
      "Average Time to Fit (s): 3.555\n",
      "Average Time to Score (s): 0.821\n"
     ]
    }
   ],
   "source": [
    "# Let's try a more basic pipeline, \n",
    "# but one that relies on selectkbest as well\n",
    "\n",
    "featurizer = CountVectorizer(ngram_range=(1, 2))\n",
    "\n",
    "select_k_text_pipe = Pipeline([('featurizer', featurizer), \n",
    "                      ('select_k', SelectKBest()),\n",
    "                      ('classify', MultinomialNB())])\n",
    "\n",
    "select_k_text_pipe_params = {'select_k__k': [1000, 5000]}\n",
    "\n",
    "get_best_model_and_accuracy(select_k_text_pipe, \n",
    "                            select_k_text_pipe_params, \n",
    "                            tweets_X, tweets_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=None, splitter='best')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# create a brand new decision tree classifier\n",
    "tree = DecisionTreeClassifier()\n",
    "\n",
    "tree.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>importance</th>\n",
       "      <th>feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.162843</td>\n",
       "      <td>PAY_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.073522</td>\n",
       "      <td>AGE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.067187</td>\n",
       "      <td>BILL_AMT1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.060263</td>\n",
       "      <td>LIMIT_BAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.054429</td>\n",
       "      <td>PAY_AMT3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    importance    feature\n",
       "5     0.162843      PAY_0\n",
       "4     0.073522        AGE\n",
       "11    0.067187  BILL_AMT1\n",
       "0     0.060263  LIMIT_BAL\n",
       "19    0.054429   PAY_AMT3"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# note that we have some other features in play besides what our last two\n",
    "# feature selection modules chose, notabley the number two feature, AGE\n",
    "\n",
    "importances = pd.DataFrame({'importance': tree.feature_importances_, 'feature':X.columns}).sort_values('importance', ascending=False)\n",
    "\n",
    "importances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 7)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# similar to SelectKBest, but not with statistical tests\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "# instantiate a class that choses features based\n",
    "# on feature importances according to the fitting phase\n",
    "# of a separate decision tree classifier\n",
    "select_from_model = SelectFromModel(DecisionTreeClassifier(), \n",
    "                                    threshold=.05)\n",
    "\n",
    "selected_X = select_from_model.fit_transform(X, y)\n",
    "\n",
    "selected_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'estimator__class_weight': None,\n",
       " 'estimator__criterion': 'gini',\n",
       " 'estimator__max_depth': None,\n",
       " 'estimator__max_features': None,\n",
       " 'estimator__max_leaf_nodes': None,\n",
       " 'estimator__min_impurity_decrease': 0.0,\n",
       " 'estimator__min_impurity_split': None,\n",
       " 'estimator__min_samples_leaf': 1,\n",
       " 'estimator__min_samples_split': 2,\n",
       " 'estimator__min_weight_fraction_leaf': 0.0,\n",
       " 'estimator__presort': False,\n",
       " 'estimator__random_state': None,\n",
       " 'estimator__splitter': 'best',\n",
       " 'estimator': DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "                        max_features=None, max_leaf_nodes=None,\n",
       "                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                        min_samples_leaf=1, min_samples_split=2,\n",
       "                        min_weight_fraction_leaf=0.0, presort=False,\n",
       "                        random_state=None, splitter='best'),\n",
       " 'max_features': None,\n",
       " 'norm_order': 1,\n",
       " 'prefit': False,\n",
       " 'threshold': 0.05}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see what the params look like so we can gridsearch them.\n",
    "select_from_model.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'd_tree' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-20f19d243892>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m select_from_pipe = Pipeline([('select', SelectFromModel(DecisionTreeClassifier())), \n\u001b[0;32m----> 7\u001b[0;31m                              ('classifier', d_tree)])\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mselect_from_pipe_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtree_pipe_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'd_tree' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# to speed things up a bit\n",
    "tree_pipe_params = {'classifier__max_depth': [1, 3, 5, 7]}\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "select_from_pipe = Pipeline([('select', SelectFromModel(DecisionTreeClassifier())), \n",
    "                             ('classifier', d_tree)])\n",
    "\n",
    "select_from_pipe_params = deepcopy(tree_pipe_params)\n",
    "\n",
    "select_from_pipe_params.update({\n",
    "              'select__threshold': [.01, .05, .1, \"mean\", \"median\", \"2.*mean\"],\n",
    "              'select__estimator__max_depth': [None, 1, 3, 5, 7]\n",
    "              })\n",
    "\n",
    "print(select_from_pipe_params)\n",
    "\n",
    "# not better than original\n",
    "get_best_model_and_accuracy(select_from_pipe, \n",
    "                            select_from_pipe_params, \n",
    "                            X, y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'select_from_pipe' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-27b306927157>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# set the optimal params to the pipeline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m select_from_pipe.set_params(**{'select__threshold': 0.01, \n\u001b[0m\u001b[1;32m      3\u001b[0m                                \u001b[0;34m'select__estimator__max_depth'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                                'classifier__max_depth': 3})\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'select_from_pipe' is not defined"
     ]
    }
   ],
   "source": [
    "# set the optimal params to the pipeline\n",
    "select_from_pipe.set_params(**{'select__threshold': 0.01, \n",
    "                               'select__estimator__max_depth': None, \n",
    "                               'classifier__max_depth': 3})\n",
    "\n",
    "\n",
    "# fit our pipeline to our data\n",
    "select_from_pipe.steps[0][1].fit(X, y)\n",
    "\n",
    "# list the columns that the SVC selected by calling the get_support() method from SelectFromModel\n",
    "X.columns[select_from_pipe.steps[0][1].get_support()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classifier__max_depth': [1, 3, 5, 7], 'select__threshold': [0.01, 0.05, 0.1, 'mean', 'median', '2.*mean'], 'select__estimator__penalty': ['l1', 'l2']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_selection/base.py:79: UserWarning: No features were selected: either the data is too noisy or the selection test too strict.\n",
      "  UserWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py:547: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to 0.000000. Details: \n",
      "ValueError: Found array with 0 feature(s) (shape=(20000, 0)) while a minimum of 1 is required.\n",
      "\n",
      "  FitFailedWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/super/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy: 0.8212\n",
      "Best Parameters: {'classifier__max_depth': 5, 'select__estimator__penalty': 'l1', 'select__threshold': 0.01}\n",
      "Average Time to Fit (s): 0.484\n",
      "Average Time to Score (s): 0.002\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from copy import deepcopy\n",
    "\n",
    "# a new selector that uses the coefficients from a regularized logistic regression as feature importances\n",
    "logistic_selector = SelectFromModel(LogisticRegression())\n",
    "\n",
    "# make a new pipeline that uses coefficients from LogistisRegressio\n",
    "# as a feature ranker\n",
    "regularization_pipe = Pipeline([('select', logistic_selector), \n",
    "                                ('classifier', tree)])\n",
    "\n",
    "regularization_pipe_params = deepcopy(tree_pipe_params)\n",
    "\n",
    "# try l1 regularization\n",
    "# and l2 regularization\n",
    "regularization_pipe_params.update({\n",
    "              'select__threshold': [.01, .05, .1, \"mean\", \"median\", \"2.*mean\"],\n",
    "              'select__estimator__penalty': ['l1', 'l2'],\n",
    "              })\n",
    "\n",
    "print(regularization_pipe_params)\n",
    "\n",
    "# better than original, in fact the best so far, and much faster on the scoring side\n",
    "get_best_model_and_accuracy(regularization_pipe, \n",
    "                            regularization_pipe_params, \n",
    "                            X, y) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# re-instantiate our Logistic Regression\n",
    "lr = LogisticRegression(penalty='l1')\n",
    "\n",
    "# fit to our data\n",
    "lr.fit(X, y)\n",
    "\n",
    "# put the coefficients on the features into a dataframe and sort\n",
    "importances = pd.DataFrame({'importance': lr.coef_.reshape(-1,), \n",
    "                            'feature':X.columns}).sort_values('importance', ascending=False)\n",
    "\n",
    "# see the best features that our pipeline chose\n",
    "importances[importances['importance'] >= .01]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVC is a linear model that uses linear supports to \n",
    "# seperate classes in euclidean space\n",
    "# This model can only work for binary classification tasks\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# Using a support vector classifier to get coefficients\n",
    "svc_selector = SelectFromModel(LinearSVC())\n",
    "\n",
    "svc_pipe = Pipeline([('select', svc_selector), \n",
    "                     ('classifier', tree)])\n",
    "\n",
    "svc_pipe_params = deepcopy(tree_pipe_params)\n",
    "\n",
    "svc_pipe_params.update({\n",
    "              'select__threshold': [.01, .05, .1, \"mean\", \"median\", \"2.*mean\"],\n",
    "              'select__estimator__penalty': ['l1', 'l2'],\n",
    "               'select__estimator__loss': ['squared_hinge', 'hinge'],\n",
    "               'select__estimator__dual': [True, False]\n",
    "              })\n",
    "\n",
    "print svc_pipe_params\n",
    "\n",
    "# better than original, in fact the best so far, and much faster on the scoring side\n",
    "get_best_model_and_accuracy(svc_pipe, \n",
    "                            svc_pipe_params, \n",
    "                            X, y) \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
